Khaganat-NeL/khanat-opennel-code
2
0
Fork 0
Code Issues 21 Pull requests Projects Releases 1 Packages Wiki Activity

Added: Basic queue for curl downloads

Browse source 
--HG--
branch : develop
This commit is contained in:
Nimetu 2017-04-16 22:01:44 +03:00
parent 9e7258d96e
commit 20c216ecda
2 changed files with 122 additions and 76 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
code/nel/include/nel/gui/group_html.h
View file

@ -820,7 +820,8 @@ namespace NLGUI
struct CDataDownload
{
public:
CDataDownload(CURL *c, const std::string &u, const std::string &d, FILE *f, TDataType t, CViewBase *i, const std::string &s, const std::string &m, const CStyleParams &style = CStyleParams()) : curl(c), url(u), dest(d), luaScript(s), md5sum(m), type(t), fp(f)
CDataDownload(const std::string &u, const std::string &d, TDataType t, CViewBase *i, const std::string &s, const std::string &m, const CStyleParams &style = CStyleParams())
: curl(NULL), fp(NULL), url(u), dest(d), type(t), luaScript(s), md5sum(m)
{
if (t == ImgType) imgs.push_back(CDataImageDownload(i, style));
}
@ -840,6 +841,8 @@ namespace NLGUI
CURLM *MultiCurl;
int RunningCurls;
bool startCurlDownload(CDataDownload &download);
void initImageDownload();
void checkImageDownload();
void addImageDownload(const std::string &url, CViewBase *img, const CStyleParams &style = CStyleParams());

193
code/nel/src/gui/group_html.cpp
View file

@ -265,6 +265,61 @@ namespace NLGUI
return dest;
}
// Add url to MultiCurl queue and return cURL handle
bool CGroupHTML::startCurlDownload(CDataDownload &download)
{
if (!MultiCurl)
{
nlwarning("Invalid MultiCurl handle, unable to download '%s'", download.url.c_str());
return false;
}
string tmpdest = download.dest + ".tmp";
// erase the tmp file if exists
if (CFile::fileExists(tmpdest))
CFile::deleteFile(tmpdest);
FILE *fp = nlfopen (tmpdest, "wb");
if (fp == NULL)
{
nlwarning("Can't open file '%s' for writing: code=%d '%s'", tmpdest.c_str (), errno, strerror(errno));
return false;
}
CURL *curl = curl_easy_init();
if (!curl)
{
fclose(fp);
CFile::deleteFile(tmpdest);
nlwarning("Creating cURL handle failed, unable to download '%s'", download.url.c_str());
return false;
}
download.curl = curl;
download.fp = fp;
curl_easy_setopt(curl, CURLOPT_NOPROGRESS, true);
curl_easy_setopt(curl, CURLOPT_URL, download.url.c_str());
// limit curl to HTTP and HTTPS protocols only
curl_easy_setopt(curl, CURLOPT_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS);
curl_easy_setopt(curl, CURLOPT_REDIR_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS);
std::string userAgent = options.appName + "/" + options.appVersion;
curl_easy_setopt(curl, CURLOPT_USERAGENT, userAgent.c_str());
sendCookies(curl, _DocumentDomain, _TrustedDomain);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, fwrite);
curl_multi_add_handle(MultiCurl, curl);
return true;
}
// Add a image download request in the multi_curl
void CGroupHTML::addImageDownload(const string &url, CViewBase *img, const CStyleParams &style)
{
@ -285,55 +340,29 @@ namespace NLGUI
// use requested url for local name
string dest = localImageName(url);
string tmpdest = localImageName(url)+".tmp";
#ifdef LOG_DL
nlwarning("add to download '%s' dest '%s' img %p", finalUrl.c_str(), dest.c_str(), img);
#endif
// erase the tmp file if exists
if (NLMISC::CFile::fileExists(tmpdest))
NLMISC::CFile::deleteFile(tmpdest);
if (!NLMISC::CFile::fileExists(dest))
{
if (!MultiCurl)
{
nlwarning("Invalid MultiCurl handle, unable to download '%s'", finalUrl.c_str());
return;
}
Curls.push_back(CDataDownload(finalUrl, dest, ImgType, img, "", "", style));
if (Curls.size() < options.curlMaxConnections) {
if (!startCurlDownload(Curls.back()))
{
Curls.pop_back();
return;
}
CURL *curl = curl_easy_init();
if (!curl)
{
nlwarning("Creating cURL handle failed, unable to download '%s'", finalUrl.c_str());
return;
}
FILE *fp = nlfopen(tmpdest, "wb");
if (fp == NULL)
{
curl_easy_cleanup(curl);
nlwarning("Can't open file '%s' for writing: code=%d '%s'", tmpdest.c_str (), errno, strerror(errno));
return;
}
curl_easy_setopt(curl, CURLOPT_NOPROGRESS, true);
curl_easy_setopt(curl, CURLOPT_URL, finalUrl.c_str());
std::string userAgent = options.appName + "/" + options.appVersion;
curl_easy_setopt(curl, CURLOPT_USERAGENT, userAgent.c_str());
sendCookies(curl, _DocumentDomain, _TrustedDomain);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, fwrite);
curl_multi_add_handle(MultiCurl, curl);
Curls.push_back(CDataDownload(curl, finalUrl, dest, fp, ImgType, img, "", "", style));
RunningCurls++;
#ifdef LOG_DL
nlwarning("adding handle %x, %d curls", curl, Curls.size());
nlwarning("(%s) adding handle %x, %d curls", _Id.c_str(), Curls.back().curl, Curls.size());
}
else
{
nlwarning("(%s) download queued, %d curls", _Id.c_str(), Curls.size());
#endif
RunningCurls++;
}
}
else
{
@ -378,15 +407,10 @@ namespace NLGUI
}
string dest = localBnpName(url);
string tmpdest = localBnpName(url)+".tmp";
#ifdef LOG_DL
nlwarning("add to download '%s' dest '%s'", url.c_str(), dest.c_str());
#endif
// erase the tmp file if exists
if (NLMISC::CFile::fileExists(tmpdest))
NLMISC::CFile::deleteFile(tmpdest);
// create/delete the local file
if (NLMISC::CFile::fileExists(dest))
{
@ -402,39 +426,23 @@ namespace NLGUI
}
if (action != "delete")
{
if (!MultiCurl)
Curls.push_back(CDataDownload(url, dest, BnpType, NULL, script, md5sum));
if (Curls.size() < options.curlMaxConnections)
{
nlwarning("Invalid MultiCurl handle, unable to download '%s'", url.c_str());
return false;
}
CURL *curl = curl_easy_init();
if (!curl)
{
nlwarning("Creating cURL handle failed, unable to download '%s'", url.c_str());
return false;
}
FILE *fp = nlfopen (tmpdest, "wb");
if (fp == NULL)
{
curl_easy_cleanup(curl);
nlwarning("Can't open file '%s' for writing: code=%d '%s'", tmpdest.c_str (), errno, strerror(errno));
return false;
}
curl_easy_setopt(curl, CURLOPT_NOPROGRESS, true);
curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, fwrite);
curl_multi_add_handle(MultiCurl, curl);
Curls.push_back(CDataDownload(curl, url, dest, fp, BnpType, NULL, script, md5sum));
if (!startCurlDownload(Curls.back()))
{
Curls.pop_back();
return false;
}
RunningCurls++;
#ifdef LOG_DL
nlwarning("adding handle %x, %d curls", curl, Curls.size());
nlwarning("(%s) adding handle %x, %d curls", _Id.c_str(), Curls.back().curl, Curls.size());
}
else
{
nlwarning("(%s) download queued, %d curls", _Id.c_str(), Curls.size());
#endif
RunningCurls++;
}
}
else
return true;
@ -631,7 +639,30 @@ namespace NLGUI
}
}
}
RunningCurls = NewRunningCurls;
if (RunningCurls < options.curlMaxConnections)
{
for (vector<CDataDownload>::iterator it=Curls.begin(); it<Curls.end(); it++)
{
if (it->curl == NULL) {
#ifdef LOG_DL
nlwarning("(%s) starting new download '%s'", _Id.c_str(), it->url.c_str());
#endif
if (!startCurlDownload(*it))
{
Curls.erase(it);
break;
}
RunningCurls++;
if (RunningCurls >= options.curlMaxConnections)
break;
}
}
}
#ifdef LOG_DL
if (RunningCurls > 0 || !Curls.empty())
nlwarning("(%s) RunningCurls %d, _Curls %d", _Id.c_str(), RunningCurls, Curls.size());
@ -4531,6 +4562,18 @@ namespace NLGUI
Curls[i].imgs.clear();
}
// remove download that are still queued
for (vector<CDataDownload>::iterator it=Curls.begin(); it<Curls.end(); )
{
if (it->curl == NULL) {
#ifdef LOG_DL
nlwarning("Remove waiting curl download (%s)", it->url.c_str());
#endif
it = Curls.erase(it);
} else {
++it;
}
}
}
// ***************************************************************************