khanat-code-old/code/nelns/admin_executor_service/log_report.cpp

// NeLNS - MMORPG Framework <http://dev.ryzom.com/projects/nel/>
// Copyright (C) 2010  Winch Gate Property Limited
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

#include "log_report.h"
#include <functional>
#include <limits>
#include "nel/misc/common.h"
#include "nel/misc/displayer.h"
#include "nel/misc/file.h"
#include "nel/misc/path.h"
#include "nel/misc/variable.h"

using namespace NLMISC;
using namespace std;


CVariable<string> LogPath( "LogReport","LogPath", "Path of the log files", ".", 0, true );

const uint MAX_LOG_LINE_SIZE = 1024;
//nlctassert(MAX_LOG_LINE_SIZE>0);

enum TLogLineHeader { LHDate, LHTime, LHType, LHThread, LHService, LHCodeFile, LHCodeLine, LHSeparator, LH_NB_FIELDS };


///
bool isLogFile( const std::string& filename )
{
	uint len = (uint)filename.size();
	return (len >= 4 ) && (filename.substr( len-4 ) == ".log");
}

///
inline bool	isNumberChar( char c )
{
	return (c >= '0') && (c <= '9');
}

///
void sortLogFiles( vector<std::string>& filenames )
{
	uint i;
	for ( i=0; i!=filenames.size(); ++i )
	{
		// Ensure that a log file without number comes *after* the ones with a number
		string name = string(filenames[i]);
		string::size_type dotpos = name.find_last_of('.');
		if ( (dotpos!=string::npos) && (dotpos > 2) )
		{
			if ( ! (isNumberChar(name[dotpos-1]) && isNumberChar(name[dotpos-2]) && isNumberChar(name[dotpos-3])) )
			{
				name = name.substr( 0, dotpos ) + "ZZZ" + name.substr( dotpos );
				filenames[i] = name.c_str();
			}
		}
	}
	sort( filenames.begin(), filenames.end() );
	for ( i=0; i!=filenames.size(); ++i )
	{
		// Set the original names back
		string name = filenames[i];
		string::size_type tokenpos = name.find( "ZZZ." );
		if ( tokenpos != string::npos )
		{
			name = name.substr( 0, tokenpos ) + name.substr( tokenpos + 3 );
			filenames[i] = name.c_str();
		}
	}
}

void	CMakeLogTask::setLogPath(const std::string & logPath)
{
	_LogPaths.resize( 1 );
	_LogPaths[0] = logPath;
}

void	CMakeLogTask::setLogPaths(const std::vector<std::string>& logPaths)
{
	_LogPaths = logPaths;
}

void	CMakeLogTask::setLogPathToDefault()
{
	setLogPath( LogPath.get() );
}

/*
 *
 */
CMakeLogTask::~CMakeLogTask()
{
	if ( _Thread ) // implies && _OutputLogReport
	{
		if ( ! _Complete )
		{
			pleaseStop();
			_Thread->wait();
		}
		clear();
	}
}


/*
 *
 */
void	CMakeLogTask::start()
{
	if ( _Thread )
	{
		if ( _Complete )
			clear();
		else
			return;
	}
	_Stopping = false;
	_Complete = false;
	_Thread = NLMISC::IThread::create( this );
	_OutputLogReport = new CLogReport();
	_Thread->start();
}


/*
 *
 */
void	CMakeLogTask::clear()
{
	if (_Thread)
	{
		delete _Thread;
		_Thread = NULL;
	}
	if (_OutputLogReport)
	{
		delete _OutputLogReport;
		_OutputLogReport = NULL;
	}
}

/*
 *
 */
void	CMakeLogTask::terminateTask()
{
	if (!_Thread) // _Thread _implies _OutputLogReport
		return;

	pleaseStop();
	_Thread->wait();

	clear();
}

//
bool isOfLogDotLogFamily( const std::string& filename )
{
	return ((filename == "log.log") ||
			 ((filename.size() == 10) &&
			  (filename.substr( 0, 3 ) == "log") &&
			  isNumberChar(filename[3]) && isNumberChar(filename[4]) && isNumberChar(filename[5]) &&
			  (filename.substr( 6, 4 ) == ".log")) );
}


enum TVersionTargetMode { TTMAll, TTMMatchAllV, TTMMatchExactV, TTMMatchGreaterV, TTMMatchLowerV } targetMode;
const uint CurrentVersion = std::numeric_limits<uint>::max();

// Return true and logVersion, or false if not a log with version
bool getLogVersion( const std::string& filename, uint& logVersion )
{
	uint len = (uint)filename.size();
	if ( (len > 4) && (filename.substr( len-4 ) == ".log") )
	{
		if ( filename.substr(0, 3) == "log" )
		{
			if ( (len == 7) ||
				 ((len == 10) && (isNumberChar(filename[3]) && isNumberChar(filename[4]) && isNumberChar(filename[5]))) )
			{
				logVersion = CurrentVersion;
				return true;
			}
		}
		else if ( filename[0] == 'v' )
		{
			string::size_type p = filename.find( "_", 1 );
			if ( p != string::npos )
			{
				if ( (len == p + 8) ||
					 ((len == p + 11) && (isNumberChar(filename[p+4]) && isNumberChar(filename[p+5]) && isNumberChar(filename[p+6]))) )
				{
					NLMISC::fromString( filename.substr( 1, p-1 ), logVersion );
					return true;
				}
			}
		}
	}
	return false;
}

// Assumes filename is .log file
bool matchLogTarget( const std::string& filename, TVersionTargetMode targetMode, uint targetVersion )
{
	if ( targetMode == TTMAll )
		return true;

	uint version;

	// Get version or exclude non-standard log files
	if ( ! getLogVersion( filename, version ) )
		return false;

	// Exclude non-matching version
	switch ( targetMode )
	{
	case TTMMatchExactV:
		return (version == targetVersion); // break;
	case TTMMatchGreaterV:
		return (version >= targetVersion); // break;
	case TTMMatchLowerV:
		return (version <= targetVersion); // break;
	default: // TTMMatchAllV
		return true;
	}
}

/*
 *
 */
void	CMakeLogTask::run()
{
	// Parse log target
	uint targetVersion = CurrentVersion;
	uint lts = (uint)_LogTarget.size();
	if ( _LogTarget.empty() || (_LogTarget == "v") )
	{
		targetMode = TTMMatchExactV;
	}
	else if ( _LogTarget == "v*" )
	{
		targetMode = TTMMatchAllV;
	}
	else if ( _LogTarget == "*" )
	{
		targetMode = TTMAll;
	}
	else if ( (lts > 1) && (_LogTarget[0] == 'v') )
	{
		uint additionalChars = 1;
		if ( _LogTarget[lts-1] == '+' )
			targetMode = TTMMatchGreaterV;
		else if ( _LogTarget[lts-1] == '-' )
			targetMode = TTMMatchLowerV;
		else
		{
			targetMode = TTMMatchExactV;
			additionalChars = 0;
		}
		
		NLMISC::fromString( _LogTarget.substr( 1, lts-additionalChars-1 ), targetVersion );
	}
	else
	{
		nlwarning( "Invalid log target argument: %s", _LogTarget.c_str() );
		_Complete = true;
		return;
	}

	// Get log files and sort them
	vector<string> filenames;
	vector<string> filenamesOfPath;
	for ( vector<string>::const_iterator ilf=_LogPaths.begin(); ilf!=_LogPaths.end(); ++ilf )
	{
		string path = (*ilf);
		if ( (! path.empty()) && (path[path.size()-1]!='/') )
			path += "/";
		filenamesOfPath.clear();
		CPath::getPathContent( path, false, false, true, filenamesOfPath, NULL, true );
		vector<string>::iterator ilf2 = partition( filenamesOfPath.begin(), filenamesOfPath.end(), isLogFile );
		filenamesOfPath.erase( ilf2, filenamesOfPath.end() );
		sortLogFiles( filenamesOfPath );
		filenames.insert( filenames.end(), filenamesOfPath.begin(), filenamesOfPath.end() );
	}

	// Analyse log files
	_OutputLogReport->reset();
	uint nbLines = 0;
	char line [MAX_LOG_LINE_SIZE];

	uint nbSkippedFiles = 0;
	for ( vector<string>::const_iterator ilf=filenames.begin(); ilf!=filenames.end(); ++ilf )
	{
		string shortname = CFile::getFilename( *ilf );

		// Filter log files based on filename before opening them
		if ( ! matchLogTarget( shortname, targetMode, targetVersion ) )
		{
			++nbSkippedFiles;
			continue;
		}

		nlinfo( "Processing %s (%u/%u)", (*ilf).c_str(), ilf-filenames.begin(), filenames.size() );
		CIFile logfile;
		if ( logfile.open( *ilf, true ) )
		{
			_OutputLogReport->setProgress( (uint)(ilf-filenames.begin()), (uint)filenames.size() );
			while ( ! logfile.eof() )
			{
				logfile.getline( line, MAX_LOG_LINE_SIZE );
				line[MAX_LOG_LINE_SIZE-1] = '\0'; // force valid end of line
				_OutputLogReport->pushLine( line );
				++nbLines;

				if ( isStopping() )
					return;
			}
		}
	}
	nlinfo( "%u lines processed", nbLines );
	if ( nbSkippedFiles != 0 )
		nlinfo( "%u log files skipped, not matching target %s", nbSkippedFiles, _LogTarget.c_str() );
	_Complete = true;
}


/*
 * Add a log line to the report tree
 */
void	CLogReport::pushLine( const std::string& line, NLMISC::CLog::TLogType onlyType, bool countOtherTypes )
{
	// Ignore session title
	if ( (line.size() > 14) && (line.substr( 0, 14 ) == "Log Starting [") )
		return;

	// Decode standard log line
	vector<string> lineTokens;
	explode( line, string(" "), lineTokens );

	if ( lineTokens.size() < LH_NB_FIELDS )
		return;

	// Filter log type
	if ( onlyType != CLog::LOG_UNKNOWN )
	{
		if ( lineTokens[LHType] != IDisplayer::logTypeToString( onlyType ) )
		{
			if ( countOtherTypes )
				storeLine( lineTokens, true );
			return;
		}
	}

	// Store
	storeLine( lineTokens, false );
}


/*
 *
 */
void	CLogReportNode::storeLine( const std::vector<std::string>& lineTokens, bool mainCountOnly )
{
	// Get service name from "[machine/]serviceName-serviceId"
	string service = lineTokens[LHService];
	string::size_type p = service.find( '/' );
	if ( p != string::npos )
		service = service.substr( p+1 );
	p = service.find( '-' );
	if ( p != string::npos )
		service = service.substr( 0, p );

	// Store to appropriate child
	CLogReportLeaf *child = getChild( service );
	if ( ! child )
		child = addChild( service );
	child->storeLine( lineTokens, mainCountOnly );
}


/*
 *
 */
void	CLogReportLeaf::storeLine( const std::vector<std::string>& lineTokens, bool mainCountOnly )
{
	if ( ! mainCountOnly )
	{
		// Build key from "codeFile codeLine"
		string key = lineTokens[LHCodeFile] + ":" + lineTokens[LHCodeLine];
		_LogLineInfo[key].addAnOccurence( lineTokens );
	}
	++_Counts[lineTokens[LHType]];
	++_TotalLines;
}


/*
 *
 */
void	CLogLineInfo::addAnOccurence( const std::vector<std::string>& lineTokens )
{
	if ( NbOccurences == 0 )
	{
		for ( uint i=LH_NB_FIELDS; i<lineTokens.size(); ++i )
		{
			if ( i != LH_NB_FIELDS )
				SampleLogText += " ";
			SampleLogText += lineTokens[i];
		}
	}
	++NbOccurences;
}


/*
 *
 */
uint	CLogReportLeaf::getNbTotalLines( NLMISC::CLog::TLogType logType )
{
	return (logType==NLMISC::CLog::LOG_UNKNOWN) ? _TotalLines : _Counts[NLMISC::IDisplayer::logTypeToString( logType )];
}


/*
 * Get results for a service
 */
void	CLogReport::reportByService( const std::string& service, NLMISC::CLog *targetLog )
{
	ILogReport *child = getChild( service );
	if ( child )
	{
		child->report( targetLog, true );
	}
	else
	{
		targetLog->displayNL( "Nothing found for service %s", service.c_str() );
	}
}


/*
 * Get results for a service (all distinct lines, sorted by occurence)
 */
void	CLogReportLeaf::report( NLMISC::CLog *targetLog, bool )
{
	// Sort it
	typedef multimap< uint, pair< string, const CLogLineInfo * >, std::greater<uint> > CSortedByOccurenceLogLineInfoMap;
	CSortedByOccurenceLogLineInfoMap sortedByOccurence;
	for ( CLogLineInfoMap::const_iterator it=_LogLineInfo.begin(); it!=_LogLineInfo.end(); ++it )
	{
		const string &key = (*it).first;
		const CLogLineInfo& info = (*it).second;
		sortedByOccurence.insert( make_pair( info.NbOccurences, make_pair( key, &info ) ) );
	}

	// Display it
	for ( CSortedByOccurenceLogLineInfoMap::const_iterator iso=sortedByOccurence.begin(); iso!=sortedByOccurence.end(); ++iso )
	{
		const string &key = (*iso).second.first;
		const CLogLineInfo& info = *((*iso).second.second);
		targetLog->displayRawNL( "%s %6u %s : %s", _Service.c_str(), info.NbOccurences, key.c_str(), info.SampleLogText.c_str() );
	}
}


/*
 * Return the number of lines displayed
 */
uint	CLogReportLeaf::reportPart( uint beginIndex, uint maxNbLines, NLMISC::CLog *targetLog )
{
	uint i = 0;
	CLogLineInfoMap::const_iterator it;
	for ( it=_LogLineInfo.begin(); it!=_LogLineInfo.end(); ++it )
	{
		if ( i >= beginIndex )
		{
			if ( i >= maxNbLines )
				return i - beginIndex;

			const string &key = (*it).first;
			const CLogLineInfo& info = (*it).second;
			targetLog->displayRawNL( "%s %6u %s : %s", _Service.c_str(), info.NbOccurences, key.c_str(), info.SampleLogText.c_str() );
		}
		++i;
	}
	return i - beginIndex;
}


/*
 * Get summary of results
 */
void	CLogReportNode::report( NLMISC::CLog *targetLog, bool displayDetailsPerService )
{
	uint nb1Sum=0, nb2Sum=0;
	for ( std::vector<CLogReportLeaf*>::const_iterator it=_Children.begin(); it!=_Children.end(); ++it )
	{
		CLogReportLeaf *pt = (*it);

		// Get distinct warnings
		uint nb1 = pt->getNbDistinctLines();
		nb1Sum += nb1;

		// Get total warnings, info... but filter out lines with no header
		uint sumTotalLinesNotUnknown = 0;
		uint nbTotalLines [CLog::LOG_UNKNOWN];
		for ( uint i=CLog::LOG_ERROR; i!=CLog::LOG_UNKNOWN; ++i )
		{
			nbTotalLines[i] = pt->getNbTotalLines( (CLog::TLogType)i );
			sumTotalLinesNotUnknown += nbTotalLines[i];
		}
		if ( sumTotalLinesNotUnknown != 0 )
		{
			targetLog->displayRawNL( "%s: \t%u distinct WRN, %u total WRN, %u INF, %u DBG, %u STT, %u AST, %u ERR, %u TOTAL",
				pt->service().c_str(), nb1, nbTotalLines[CLog::LOG_WARNING],
				nbTotalLines[CLog::LOG_INFO], nbTotalLines[CLog::LOG_DEBUG],
				nbTotalLines[CLog::LOG_STAT], nbTotalLines[CLog::LOG_ASSERT],
				nbTotalLines[CLog::LOG_ERROR], pt->getNbTotalLines( CLog::LOG_UNKNOWN ) );
			nb2Sum += nbTotalLines[CLog::LOG_WARNING];
		}
	}
	targetLog->displayRawNL( "=> %u distinct, %u total WRN (%u pages)", nb1Sum, nb2Sum, nb1Sum / NB_LINES_PER_PAGE + 1 );
	
	if ( displayDetailsPerService )
	{
		for ( std::vector<CLogReportLeaf*>::const_iterator it=_Children.begin(); it!=_Children.end(); ++it )
		{
			(*it)->report( targetLog, true );
		}
	}
}


/*
 * Get partial results (pageNum>=1)
 */
void	CLogReportNode::reportPage( uint pageNum, NLMISC::CLog *targetLog )
{
	if ( _Children.empty() )
	{
		targetLog->displayRawNL( "[END OF LOG]" );
		return;
	}

	uint beginIndex = pageNum * NB_LINES_PER_PAGE;
	uint lineCounter = 0, prevLineCounter;
	for ( std::vector<CLogReportLeaf*>::const_iterator it=_Children.begin(); it!=_Children.end(); ++it )
	{
		CLogReportLeaf *pt = (*it);
		prevLineCounter = lineCounter;
		lineCounter += pt->getNbDistinctLines();
		if ( lineCounter >= beginIndex )
		{
			uint remainingLines = pageNum - (lineCounter - beginIndex );
			pt->reportPart( beginIndex - prevLineCounter, remainingLines, targetLog ); //
			while ( remainingLines != 0 )
			{
				++it;
				if ( it == _Children.end() )
				{
					targetLog->displayRawNL( "[END OF LOG]" );
					return;
				}
				pt = (*it);
				remainingLines -= pt->reportPart( 0, remainingLines, targetLog );
			}
			return;
		}
	}	
}