Khaganat-NeL/khanat-opennel-code
2
0
Fork 0
Code Issues 21 Pull requests Projects Releases 1 Packages Wiki Activity

Merge with develop

Browse source 
--HG--
branch : compatibility-develop
This commit is contained in:
Nimetu 2017-04-25 00:23:09 +03:00
commit eb23a3b61c
8 changed files with 382 additions and 29 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
code/nel/include/nel/gui/group_html.h
View file

@ -821,13 +821,13 @@ namespace NLGUI
{
public:
CDataDownload(const std::string &u, const std::string &d, TDataType t, CViewBase *i, const std::string &s, const std::string &m, const CStyleParams &style = CStyleParams())
: curl(NULL), fp(NULL), url(u), dest(d), type(t), luaScript(s), md5sum(m)
: data(NULL), fp(NULL), url(u), dest(d), type(t), luaScript(s), md5sum(m)
{
if (t == ImgType) imgs.push_back(CDataImageDownload(i, style));
}
public:
CURL *curl;
CCurlWWWData *data;
std::string url;
std::string dest;
std::string luaScript;

77
code/nel/include/nel/gui/http_cache.h Normal file
View file

@ -0,0 +1,77 @@
// Ryzom - MMORPG Framework <http://dev.ryzom.com/projects/ryzom/>
// Copyright (C) 2010 Winch Gate Property Limited
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
#ifndef CL_HTTP_CACHE_H
#define CL_HTTP_CACHE_H
#include "nel/misc/types_nl.h"
namespace NLGUI
{
struct CHttpCacheObject
{
CHttpCacheObject(uint32 expires = 0, const std::string& lastModified = "", const std::string& etag = "")
: Expires(expires)
, LastModified(lastModified)
, Etag(etag){};
uint32 Expires;
std::string LastModified;
std::string Etag;
void serial(NLMISC::IStream& f);
};
/**
* Keeping track of downloaded files cache related headers
* \author Meelis Mägi (nimetu)
* \date 2017
*/
class CHttpCache
{
typedef std::map<std::string, CHttpCacheObject> THttpCacheMap;
public:
static CHttpCache* getInstance();
static void release();
public:
void setCacheIndex(const std::string& fname);
void init();
CHttpCacheObject lookup(const std::string& fname);
void store(const std::string& fname, const CHttpCacheObject& data);
void flushCache();
void serial(NLMISC::IStream& f);
private:
CHttpCache();
~CHttpCache();
void pruneCache();
static CHttpCache* instance;
THttpCacheMap _List;
std::string _IndexFilename;
bool _Initialized;
size_t _MaxObjects;
};
}
#endif

3
code/nel/src/3d/meshvp_wind_tree.cpp
View file

@ -226,7 +226,8 @@ void CMeshVPWindTree::initVertexPrograms()
{
// setup of the VPLight fragment
uint numPls= i/4;
bool normalize= (i&1)!=0;
// FIXME: normalize=true makes trees dance, workaround for issue #160
bool normalize= false; //(i&1)!=0;
bool specular= (i&2)!=0;
// combine

118
code/nel/src/gui/group_html.cpp
View file

@ -45,6 +45,7 @@
#include "nel/3d/texture_file.h"
#include "nel/misc/big_file.h"
#include "nel/gui/url_parser.h"
#include "nel/gui/http_cache.h"
using namespace std;
using namespace NLMISC;
@ -87,6 +88,15 @@ namespace NLGUI
curl_slist_free_all(HeadersSent);
}
void sendHeaders(const std::vector<std::string> headers)
{
for(uint i = 0; i < headers.size(); ++i)
{
HeadersSent = curl_slist_append(HeadersSent, headers[i].c_str());
}
curl_easy_setopt(Request, CURLOPT_HTTPHEADER, HeadersSent);
}
void setRecvHeader(const std::string &header)
{
size_t pos = header.find(": ");
@ -110,12 +120,42 @@ namespace NLGUI
return "";
}
const uint32 getExpires()
{
time_t ret = 0;
if (HeadersRecv.count("expires") > 0)
ret = curl_getdate(HeadersRecv["expires"].c_str(), NULL);
return ret > -1 ? ret : 0;
}
const std::string getLastModified()
{
if (HeadersRecv.count("last-modified") > 0)
{
return HeadersRecv["last-modified"];
}
return "";
}
const std::string getEtag()
{
if (HeadersRecv.count("etag") > 0)
{
return HeadersRecv["etag"];
}
return "";
}
public:
CURL *Request;
std::string Url;
std::string Content;
private:
// headers sent with curl request, must be released after transfer
curl_slist * HeadersSent;
@ -274,19 +314,16 @@ namespace NLGUI
return false;
}
// TODO: replace with expire and etag headers
if (CFile::fileExists(download.dest))
time_t currentTime;
time(&currentTime);
CHttpCacheObject cache = CHttpCache::getInstance()->lookup(download.dest);
if (cache.Expires > currentTime)
{
time_t currentTime;
time(&currentTime);
uint32 mtime = CFile::getFileModificationDate(download.dest);
if (mtime + 3600 > currentTime)
{
#ifdef LOG_DL
nlwarning("Cache for (%s) is not expired (%s, age:%d)", download.url.c_str(), download.dest.c_str(), currentTime - mtime);
nlwarning("Cache for (%s) is not expired (%s, expires:%d)", download.url.c_str(), download.dest.c_str(), cache.Expires - currentTime);
#endif
return false;
}
return false;
}
string tmpdest = download.dest + ".tmp";
@ -312,7 +349,7 @@ namespace NLGUI
return false;
}
download.curl = curl;
download.data = new CCurlWWWData(curl, download.url);
download.fp = fp;
curl_easy_setopt(curl, CURLOPT_NOPROGRESS, true);
@ -322,6 +359,20 @@ namespace NLGUI
curl_easy_setopt(curl, CURLOPT_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS);
curl_easy_setopt(curl, CURLOPT_REDIR_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS);
std::vector<std::string> headers;
if (!cache.Etag.empty())
headers.push_back("If-None-Match: " + cache.Etag);
if (!cache.LastModified.empty())
headers.push_back("If-Modified-Since: " + cache.LastModified);
if (headers.size() > 0)
download.data->sendHeaders(headers);
// catch headers
curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, curlHeaderCallback);
curl_easy_setopt(curl, CURLOPT_WRITEHEADER, download.data);
std::string userAgent = options.appName + "/" + options.appVersion;
curl_easy_setopt(curl, CURLOPT_USERAGENT, userAgent.c_str());
@ -369,7 +420,7 @@ namespace NLGUI
RunningCurls++;
#ifdef LOG_DL
nlwarning("(%s) adding handle %x, %d curls", _Id.c_str(), Curls.back().curl, Curls.size());
nlwarning("(%s) adding handle %x, %d curls", _Id.c_str(), Curls.back().data->Request, Curls.size());
}
else
{
@ -443,7 +494,7 @@ namespace NLGUI
}
RunningCurls++;
#ifdef LOG_DL
nlwarning("(%s) adding handle %x, %d curls", _Id.c_str(), Curls.back().curl, Curls.size());
nlwarning("(%s) adding handle %x, %d curls", _Id.c_str(), Curls.back().data->Request, Curls.size());
}
else
{
@ -580,26 +631,44 @@ namespace NLGUI
for (vector<CDataDownload>::iterator it=Curls.begin(); it<Curls.end(); it++)
{
if(msg->easy_handle == it->curl)
if(it->data && it->data->Request == msg->easy_handle)
{
CURLcode res = msg->data.result;
long r;
curl_easy_getinfo(it->curl, CURLINFO_RESPONSE_CODE, &r);
curl_easy_getinfo(it->data->Request, CURLINFO_RESPONSE_CODE, &r);
fclose(it->fp);
receiveCookies(it->data->Request, _DocumentDomain, _TrustedDomain);
#ifdef LOG_DL
nlwarning("(%s) transfer '%p' completed with status %d, http %d, url (len %d) '%s'", _Id.c_str(), it->curl, res, r, it->url.size(), it->url.c_str());
nlwarning("(%s) transfer '%p' completed with status %d, http %d, url (len %d) '%s'", _Id.c_str(), it->data->Request, res, r, it->url.size(), it->url.c_str());
#endif
curl_multi_remove_handle(MultiCurl, it->curl);
curl_easy_cleanup(it->curl);
curl_multi_remove_handle(MultiCurl, it->data->Request);
string tmpfile = it->dest + ".tmp";
if(res != CURLE_OK || r < 200 || r >= 300 || (!it->md5sum.empty() && (it->md5sum != getMD5(tmpfile).toString())))
{
NLMISC::CFile::deleteFile(tmpfile.c_str());
// 304 Not Modified
if (res == CURLE_OK && r == 304)
{
CHttpCacheObject obj;
obj.Expires = it->data->getExpires();
obj.Etag = it->data->getEtag();
obj.LastModified = it->data->getLastModified();
CHttpCache::getInstance()->store(it->dest, obj);
}
}
else
{
CHttpCacheObject obj;
obj.Expires = it->data->getExpires();
obj.Etag = it->data->getEtag();
obj.LastModified = it->data->getLastModified();
CHttpCache::getInstance()->store(it->dest, obj);
string finalUrl;
if (it->type == ImgType)
{
@ -642,6 +711,9 @@ namespace NLGUI
}
}
// release CCurlWWWData
delete it->data;
Curls.erase(it);
break;
}
@ -656,7 +728,7 @@ namespace NLGUI
{
for (vector<CDataDownload>::iterator it=Curls.begin(); it<Curls.end(); it++)
{
if (it->curl == NULL) {
if (it->data == NULL) {
#ifdef LOG_DL
nlwarning("(%s) starting new download '%s'", _Id.c_str(), it->url.c_str());
#endif
@ -4575,7 +4647,7 @@ namespace NLGUI
// remove download that are still queued
for (vector<CDataDownload>::iterator it=Curls.begin(); it<Curls.end(); )
{
if (it->curl == NULL) {
if (it->data == NULL) {
#ifdef LOG_DL
nlwarning("Remove waiting curl download (%s)", it->url.c_str());
#endif
@ -5172,11 +5244,7 @@ namespace NLGUI
std::vector<std::string> headers;
headers.push_back("Accept-Language: "+options.languageCode);
headers.push_back("Accept-Charset: utf-8");
for(uint i=0; i< headers.size(); ++i)
{
_CurlWWW->HeadersSent = curl_slist_append(_CurlWWW->HeadersSent, headers[i].c_str());
}
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, _CurlWWW->HeadersSent);
_CurlWWW->sendHeaders(headers);
// catch headers for redirect
curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, curlHeaderCallback);

200
code/nel/src/gui/http_cache.cpp Normal file
View file

@ -0,0 +1,200 @@
// Ryzom - MMORPG Framework <http://dev.ryzom.com/projects/ryzom/>
// Copyright (C) 2010 Winch Gate Property Limited
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
#include "stdpch.h"
#include "nel/gui/http_cache.h"
using namespace std;
using namespace NLMISC;
#ifdef DEBUG_NEW
#define new DEBUG_NEW
#endif
namespace NLGUI
{
CHttpCache* CHttpCache::instance = NULL;
CHttpCache* CHttpCache::getInstance()
{
if (!instance)
{
instance = new CHttpCache();
}
return instance;
}
void CHttpCache::release()
{
delete instance;
instance = NULL;
}
CHttpCache::CHttpCache()
: _Initialized(false)
, _MaxObjects(100)
{ };
CHttpCache::~CHttpCache()
{
flushCache();
}
void CHttpCache::setCacheIndex(const std::string& fname)
{
_IndexFilename = fname;
_Initialized = false;
}
CHttpCacheObject CHttpCache::lookup(const std::string& fname)
{
if (!_Initialized)
init();
if (_List.count(fname) > 0)
return _List[fname];
return CHttpCacheObject();
}
void CHttpCache::store(const std::string& fname, const CHttpCacheObject& data)
{
if (!_Initialized)
init();
_List[fname] = data;
}
void CHttpCache::init()
{
if (_Initialized)
return;
_Initialized = true;
if (_IndexFilename.empty() || !CFile::fileExists(_IndexFilename))
return;
CIFile in;
if (!in.open(_IndexFilename)) {
nlwarning("Unable to open %s for reading", _IndexFilename.c_str());
return;
}
serial(in);
}
void CHttpCacheObject::serial(NLMISC::IStream& f)
{
f.serialVersion(1);
f.serial(Expires);
f.serial(LastModified);
f.serial(Etag);
}
void CHttpCache::serial(NLMISC::IStream& f)
{
// saved state is ignored when version checks fail
try {
f.serialVersion(1);
// CacheIdx
f.serialCheck(NELID("hcaC"));
f.serialCheck(NELID("xdIe"));
if (f.isReading())
{
uint32 numFiles;
f.serial(numFiles);
_List.clear();
for (uint k = 0; k < numFiles; ++k)
{
std::string fname;
f.serial(fname);
CHttpCacheObject obj;
obj.serial(f);
_List[fname] = obj;
}
}
else
{
uint32 numFiles = _List.size();
f.serial(numFiles);
for (THttpCacheMap::iterator it = _List.begin(); it != _List.end(); ++it)
{
std::string fname(it->first);
f.serial(fname);
(*it).second.serial(f);
}
}
} catch (...) {
_List.clear();
nlwarning("Invalid cache index format (%s)", _IndexFilename.c_str());
return;
}
}
void CHttpCache::pruneCache()
{
if (_List.size() < _MaxObjects)
return;
size_t mustDrop = _List.size() - _MaxObjects;
time_t currentTime;
time(&currentTime);
// if we over object limit, then start removing expired objects
// this does not guarantee that max limit is reached
for (THttpCacheMap::iterator it = _List.begin(); it != _List.end();) {
if (it->second.Expires <= currentTime) {
it = _List.erase(it);
--mustDrop;
if (mustDrop == 0)
break;
}
else
{
++it;
}
}
}
void CHttpCache::flushCache()
{
if (_IndexFilename.empty())
return;
pruneCache();
COFile out;
if (!out.open(_IndexFilename))
{
nlwarning("Unable to open %s for writing", _IndexFilename.c_str());
return;
}
serial(out);
out.close();
}
}

1
code/nel/src/gui/stdpch.h
View file

@ -26,6 +26,7 @@
#include <string>
#include <limits>
#include <ctime>
#include "nel/misc/types_nl.h"
#include "nel/misc/algo.h"

5
code/ryzom/client/src/init.cpp
View file

@ -68,6 +68,7 @@
#include "interface_v3/sbrick_manager.h"
#include "nel/gui/widget_manager.h"
#include "nel/gui/http_cache.h"
//
#include "gabarit.h"
#include "hair_set.h"
@ -1365,6 +1366,10 @@ void prelogInit()
//nlinfo ("PROFILE: %d seconds for Add search paths Data", (uint32)(ryzomGetLocalTime ()-initPaths)/1000);
}
// Initialize HTTP cache
CHttpCache::getInstance()->setCacheIndex("cache/cache.index");
CHttpCache::getInstance()->init();
// Register the reflected classes
registerInterfaceElements();

3
code/ryzom/client/src/release.cpp
View file

@ -94,7 +94,7 @@
#include "bg_downloader_access.h"
#include "nel/gui/lua_manager.h"
#include "item_group_manager.h"
#include "nel/gui/http_cache.h"
///////////
// USING //
@ -687,6 +687,7 @@ void release()
CWidgetManager::release();
CViewRenderer::release();
CIXml::releaseLibXml();
CHttpCache::release();
#if FINAL_VERSION
// openURL ("http://ryzom.com/exit/");