clientbot/client.py

#!/usr/bin/python3
# -*- coding: utf-8 -*-
#
# script to emulate client khanat
#
#    Copyright (C) 2019  AleaJactaEst
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.

# Ex.: ./client.py --khanat-host 172.17.0.3 -d --size-buffer-file 10241024

# Modifier les droits pour les nouveaux joueurs (accès à tout)
# mysql -u root -e "use nel_ams_lib;UPDATE settings SET Value = 7 WHERE settings.Setting = 'Domain_Auto_Add';"

import argparse
import http.client
import crypt
import logging
import os
import os.path
import sys
import urllib.request
import urllib.parse
import tempfile
from enum import IntEnum
from ctypes import *
import re
import random
import lzma
import socket
import struct
import xml.etree.ElementTree as ET
import hashlib
import time
import signal

INVALID_SLOT = 0xff

class BitStream():
    def __init__(self):
        self._pos = 0
        self._read = 0
        self._tampon = []
        self._groupRead = []

    def __len__(self):
        return (self._pos  + 7) // 8

    def needRead(self):
        return self._pos - self._read

    def sizeData(self):
        return self._pos

    def sizeRead(self):
        return self._read

    def getRead(self):
        return self._read

    def getPos(self):
        return self._pos

    def putRead(self, value):
        if value > self._pos:
            raise ValueError
        self._read = value

    # ------------------------------------
    def internalSerial(self, value, nbits):
        if nbits == 0:
            return
        elif nbits > 32:
            raise "Out of range"
        pos = self._pos % 8
        if pos == 0:
            self._tampon.append(0)
        # print(">", pos, value)
        value = c_uint32(value).value
        if nbits != 32:
            mask = (1 << nbits) - 1;
            v = value & mask;
        else:
            v = value;
        _FreeBits = 8 - (self._pos % 8)
        if nbits > _FreeBits:
            #print(">A")
            self._tampon[-1] |= (v >> ( nbits - _FreeBits))
            self._pos += _FreeBits
            self.internalSerial( v , nbits - _FreeBits)
        else:
            #print(">B")
            self._tampon[-1] |= (v << ( _FreeBits - nbits))
            self._pos += nbits

    def pushBool(self, valeur):
        if valeur:
            v = 1
        else:
            v = 0
        self.internalSerial(v, 1)

    def pushUint32(self, valeur):
        self.internalSerial(valeur, 32)

    def pushSint32(self, valeur):
        self.internalSerial(valeur, 32)

    def pushUint16(self, valeur):
        self.internalSerial(valeur, 16)

    def pushSint16(self, valeur):
        self.internalSerial(valeur, 16)

    def pushUint8(self, valeur):
        self.internalSerial(valeur, 8)

    def pushSint8(self, valeur):
        self.internalSerial(valeur, 8)

    def pushUint64(self, valeur):
        self.internalSerial(valeur, 32)
        self.internalSerial(valeur >> 32, 32)

    def pushSint64(self, valeur):
        self.internalSerial(valeur, 32)
        self.internalSerial(valeur >> 32, 32)

    def pushFloat(self, valeur):
        v = c_float(valeur).value
        v1 = struct.pack('f', v)
        v2 = struct.unpack('<i',v1)[0]
        self.internalSerial(v2, 32)

    def pushDouble(self, valeur):
        v = c_double(valeur).value
        v1 = struct.pack('d', v)
        v2 = struct.unpack('<Q',v1)[0]
        #self.internalSerial(v2, 32)
        self.internalSerial(v2, 32)
        self.internalSerial(v2 >> 32, 32)

    def pushChar(self, valeur):
        v = ord(valeur)
        self.internalSerial(v, 8)

    def pushString(self, valeur):
        #size=len(valeur)
        #self.internalSerial(size, 32)
        self.pushUint32(len(valeur))
        for x in valeur:
            self.pushChar(x)
            #y = ord(x)
            #self.internalSerial(y, 8)
    def pushArrayUint8(self, valeur):
        ' ex.: pushArrayChar([0,1,3,4]) '
        for i in valeur:
            self.pushUint8(i)

    def pushBitStream(self, source):
        srcRead = source.getRead()
        source.putRead(0)
        need = 8 - (self._pos % 8)
        if need != 8:
            self.internalSerial(source.readSerial(need), need)
        while source.needRead() >= 8:
            self.pushUint8(source.readSerial(8, False))

        need = source.needRead()
        if need > 0:
            self.internalSerial(source.readSerial(need, False), need)

        source.putRead(srcRead)

    # ------------------------------------
    def readSerial(self, nbits, name="", decode=True, typeName='', emulate=False):
        if nbits == 0:
            return
        elif nbits > 32:
            raise "Out of range"
        if self._read + nbits > self._pos:
            raise "Stream Overflow"
        if decode and not emulate:
            self._groupRead.append((self._read, self._read+nbits, name, typeName))
        if emulate:
            oldRead = self._read
        value = 0
        pos = self._read // 8
        _FreeBits = 8 - (self._read % 8)
        v = self._tampon[pos] & ((1 << _FreeBits) - 1)
        if nbits > _FreeBits:
            value |= (v << (nbits-_FreeBits))
            self._read += _FreeBits
            value |= self.readSerial(nbits - _FreeBits, decode=False)
        else:
            value |= (v >> (_FreeBits-nbits))
            self._read += nbits
        if emulate:
            self._read = oldRead
        return value

    def readBool(self, name):
        v = self.readSerial(1, name=name, typeName='Bool')
        if v != 0:
            return True
        else:
            return False

    def readUint32(self, name, decode=True, typeName='Uint32'):
        v = self.readSerial(32, name=name, decode=decode, typeName=typeName)
        return v

    def readSint32(self, name):
        v = self.readSerial(32, name=name, typeName='Sint32')
        return c_int32(v).value

    def readUint16(self, name):
        v = self.readSerial(16, name=name, typeName='Uint16')
        return v

    def readSint16(self, name):
        v = self.readSerial(16, name=name, typeName='Sint16')
        return c_int16(v).value

    def readUint8(self, name, decode=True, typeName='Uint8'):
        v = self.readSerial(8, decode=decode, name=name, typeName=typeName)
        return v

    def readSint8(self, name):
        v = self.readSerial(8, name=name, typeName='Sint8')
        return c_int8(v).value

    def readUint64(self, name):
        self._groupRead.append((self._read, self._read+64, name, 'Uint64'))
        v = self.readSerial(32, decode=False)
        v1 = self.readSerial(32, decode=False)
        v2 = v | (v1 << 32)
        return v2

    def readSint64(self, name):
        self._groupRead.append((self._read, self._read+64, name, 'Sint64'))
        v = self.readSerial(32, decode=False)
        v1 = self.readSerial(32, decode=False)
        v2 = v | (v1 << 32)
        return c_int64(v2).value

    def readFloat(self, name):
        v = self.readSerial(32, name=name, typeName='Float')
        v1 = struct.pack('I', v)
        v2 = struct.unpack('<f',v1)[0]
        return v2

    def readDouble(self, name):
        self._groupRead.append((self._read, self._read+64, name, 'Double'))
        v = self.readSerial(32, decode=False)
        v1 = struct.pack('I', v)
        w = self.readSerial(32, decode=False)
        w1 = struct.pack('I', w)
        x = v1 + w1
        x1 = struct.unpack('<d', x)[0]
        return x1

    def readChar(self, name, decode=True):
        v = self.readUint8(name=name, decode=decode, typeName='Char')
        return chr(v)

    def readString(self, name):
        tmp = ''
        v1 = self._read
        _size = self.readUint32('', decode=False)
        while _size > 0:
            x = self.readChar('', decode=False)
            tmp += x
            _size -= 1
        v2 = self._read
        if v2 > self._pos:
            raise ValueError
        self._groupRead.append((v1, v2, name, 'String'))
        return tmp

    def readArrayUint8(self, size, name):
        ret = []
        v1 = self._read
        for i in range(0, size):
            ret.append(self.readUint8('', decode=False))
        v2 = self._read
        self._groupRead.append((v1, v2, name, 'ArrayUint8'))
        return ret

    def readBitStreamUint8(self, size, name):
        ret = BitStream()
        v1 = self._read
        for i in range(0, size):
            ret.pushUint8(self.readUint8('', decode=False))
        v2 = self._read
        self._groupRead.append((v1, v2, name, 'StreamUint8'))
        return ret

    # ------------------------------------
    def __str__(self):
        return ''.join([ chr(x) for x in self._tampon])

    def message(self):
        # return str(self._pos) + ':' + '.'.join([ format(x, "02x") for x in self._tampon])
        return str(self._pos) + ':' + '.'.join([ "{0:08b}".format(x) for x in self._tampon])

    def toBytes(self):
        return bytes( self._tampon )

    def fromBytes(self, data):
        self._read = 0
        self._tampon = [int(x) for x in data]
        self._pos = len(self._tampon) * 8

    def showLastData(self):
        ret = ""
        readBefore = self._read
        while self._read < self._pos:
            if self._pos - self._read >= 8:
                nsize = 8
            else:
                nsize = self._pos - self._read
            data = self.readSerial(nsize, decode=False)
            if nsize == 1:
                ret += "{0:01b}".format(data)
            elif nsize == 2:
                ret += "{0:02b}".format(data)
            elif nsize == 3:
                ret += "{0:03b}".format(data)
            elif nsize == 4:
                ret += "{0:04b}".format(data)
            elif nsize == 5:
                ret += "{0:05b}".format(data)
            elif nsize == 6:
                ret += "{0:06b}".format(data)
            elif nsize == 7:
                ret += "{0:07b}".format(data)
            else:
                ret += "{0:08b}".format(data)
            if ret != "":
                ret += "."
            ret += "{0:08b}".format(data)
        self._read = readBefore
        return ret

    def showAllData(self):
        ret = ""
        readBefore = self._read
        self._read = 0
        while self._read < self._pos:
            if self._pos - self._read >= 8:
                nsize = 8
            else:
                nsize = self._pos - self._read
            data = self.readSerial(nsize, decode=False)
            if nsize == 1:
                ret += "{0:01b}".format(data)
            elif nsize == 2:
                ret += "{0:02b}".format(data)
            elif nsize == 3:
                ret += "{0:03b}".format(data)
            elif nsize == 4:
                ret += "{0:04b}".format(data)
            elif nsize == 5:
                ret += "{0:05b}".format(data)
            elif nsize == 6:
                ret += "{0:06b}".format(data)
            elif nsize == 7:
                ret += "{0:07b}".format(data)
            else:
                ret += "{0:08b}".format(data)
        self._read = readBefore
        ret2 = ""

        last = 0
        for x, y, name, typeName in self._groupRead:
            ret2 += "[<" + str(x) + ':' + str(y-1) + "> " + str(name) + ' (' + typeName + ') = ' + ret[x:y] + "]"
            last = y
        if last < self._pos:
            ret2 += "{" + ret[last:] + "}"

        return ret2
    def showAllDataRaw(self):
        ret = ""
        readBefore = self._read
        self._read = 0
        while self._read < self._pos:
            if self._pos - self._read >= 8:
                nsize = 8
            else:
                nsize = self._pos - self._read
            data = self.readSerial(nsize, decode=False)
            if nsize == 1:
                ret += "{0:01b}".format(data)
            elif nsize == 2:
                ret += "{0:02b}".format(data)
            elif nsize == 3:
                ret += "{0:03b}".format(data)
            elif nsize == 4:
                ret += "{0:04b}".format(data)
            elif nsize == 5:
                ret += "{0:05b}".format(data)
            elif nsize == 6:
                ret += "{0:06b}".format(data)
            elif nsize == 7:
                ret += "{0:07b}".format(data)
            else:
                ret += "{0:08b}".format(data)
            if ret != "":
                ret += "."
        self._read = readBefore
        return ret

    def getPosInBit(self):
        return self._pos

def TestBitStream():
    a = BitStream()
    a.pushBool(True)
    a.pushBool(False)
    a.pushBool(True)
    a.pushBool(True)
    a.pushUint32(1234567890)
    a.pushSint32(-1234567890)
    a.pushUint16(12345)
    a.pushSint16(-12345)
    a.pushUint8(123)
    a.pushSint8(-123)
    a.pushFloat(-3.3999999521443642e+38) #-3.4E+38) # 1.2339999675750732)
    a.pushDouble(-1.7E+308)
    a.pushUint64(16045690709418696365)
    a.pushSint64(-1)
    a.pushChar('a')
    a.pushString("Test A Faire")
    print('raw:')
    print(a.showAllDataRaw())
    print("-" * 20)
    print("-" * 80)
    print(a.readBool('a1'))
    print(a.readBool('a2'))
    print(a.readBool('a3'))
    print(a.readBool('a4'))
    print(a.readUint32('a5'))
    print(a.readSint32('a6'))
    print(a.readUint16('a7'))
    print(a.readSint16('a8'))
    print(a.readUint8('a9'))
    print(a.readSint8('a10'))
    print(a.readFloat('a11'))
    print(a.readDouble('a12'))
    print(a.readUint64('a13'))
    print(a.readSint64('a14'))
    print(a.readChar('a15'))
    print(a.readString('a16'))
    print(a.toBytes())
    print("-" * 40)
    print(a.showAllData())
    print("-" * 80)
    b = BitStream()
    b.fromBytes(a.toBytes())
    print(b.readBool('b1'))
    print(b.readBool('b2'))
    print(b.readBool('b3'))
    print(b.readBool('b4'))
    print(b.readUint32('b5'))
    print(b.readSint32('b6'))
    print(b.readUint16('b7'))
    print(b.readSint16('b8'))
    print(b.readUint8('b9'))
    print(b.readSint8('b10'))
    print(b.readFloat('b11'))
    print(b.readDouble('b12'))
    print(b.readUint64('b13'))
    print(b.readSint64('b14'))
    print(b.readChar('b15'))
    print(b.readString('b16'))
    print(b.toBytes())
    print("-" * 40)
    print(b.showAllData())
    print("-" * 80)
    c = BitStream()
    c.pushBool(True)
    c.pushBitStream(a)
    c.pushBitStream(b)
    print(c.readBool('c1'))
    print("-" * 80)
    print(c.readBool('c2'))
    print(c.readBool('c3'))
    print(c.readBool('c4'))
    print(c.readBool('c5'))
    print(c.readUint32('c6'))
    print(c.readSint32('c7'))
    print(c.readUint16('c8'))
    print(c.readSint16('c9'))
    print(c.readUint8('c10'))
    print(c.readSint8('c11'))
    print(c.readFloat('c12'))
    print(c.readDouble('c13'))
    print(c.readUint64('c14'))
    print(c.readSint64('c15'))
    print(c.readChar('c16'))
    print(c.readString('c17'))
    print(c.toBytes())
    print("-" * 50)
    print(c.showAllData())
    print("-" * 50)
    print(c.readBool('c18'))
    print(c.readBool('c19'))
    print(c.readBool('c20'))
    print(c.readBool('c21'))
    print(c.readUint32('c22'))
    print(c.readSint32('c23'))
    print(c.readUint16('c24'))
    print(c.readSint16('c25'))
    print(c.readUint8('c26'))
    print(c.readSint8('c27'))
    print(c.readFloat('c28'))
    print(c.readDouble('c29'))
    print(c.readUint64('c30'))
    print(c.readSint64('c31'))
    print(c.readChar('c32'))
    print("-" * 40)
    print(c.showAllData())
    print(c.readString('c33'))
    print("-" * 40)
    print(c.showAllData())
    print(c.toBytes())
    print("-" * 40)
    print(c.showAllDataRaw())
    print("-" * 20)
    print(c.showAllData())
    print("-" * 80)

NL_BITLEN = 32
class CBitSet:
    def __init__(self):
        self.data = self.resize(1024)
        self.NumBits = 0
        self.MaskLast = 0

    def resize(self, numBits):
        self.data = [ 0 for _ in range(0, (numBits +NL_BITLEN - 1) // NL_BITLEN) ]
        self.NumBits = numBits
        nLastBits = self.NumBits & (NL_BITLEN-1)
        if nLastBits == 0:
            self.MaskLast = ~0
        else:
            self.MaskLast = (1 << nLastBits)-1
        self.clearAll()

    def clearData(self):
        self.data = []
        self.NumBits = 0
        self.MaskLast = 0

    def clearAll(self):
        for i in range(0, len(self.data)):
            self.data[i] = 0

    def set(self, bitNumber, value):
        mask = bitNumber & (NL_BITLEN-1)
        mask = 1 << mask
        if value:
            self.data[bitNumber >> 5] |= mask
        else:
            self.data[bitNumber >> 5] &= ~mask

    def get(self, bitNumber):
        mask= bitNumber&(NL_BITLEN-1);
        mask= 1<<mask;
        return self.data[bitNumber >> 5] & mask != 0

    def setBit(self, bitNumber):
        self.set(bitNumber, True)

    def clearBit(self, bitNumber):
        self.set(bitNumber, False)

    def __str__(self):
        return '.'.join([hex(x) for x in self.data])
    def writeSerial(self, msgout):
#        v = 0 # currentVersion
#        if v >= 0xff:
#            msgout.pushUint8(0xff)
#            msgout.pushUint8(v)
#        else:
#            msgout.pushUint8(v)
        msgout.pushUint8(0) # currentVersion =0
        msgout.pushUint32(self.NumBits)
        msgout.pushUint32(len(self.data)) # il est lié à 'self.NumBits' dommage que l'on envoie celui-la
        for x in self.data:
            msgout.pushUint32(x)

def TestCBitSet():
    cBitSet = CBitSet()
    cBitSet.resize(1024)
    cBitSet.set(1, True)
    cBitSet.set(3, True)
    cBitSet.set(2, False)
    cBitSet.set(13, True)
    cBitSet.set(128, True)
    cBitSet.set(1023, True)
    print(cBitSet)
    print(cBitSet.get(3))
    cBitSet.set(3, False)
    print(cBitSet)
    print(cBitSet.get(3))

def getTextMD5(dataRawXml):
    log = logging.getLogger('myLogger')
    dataNew = ''
    for data in dataRawXml:
        if data != '\r': # '\015' ignore caractère \r\n =>
            dataNew += data
        else:
            log.debug("***** data:%d" % (ord(data)))
    m = hashlib.md5()
    m.update(dataNew.encode('utf-8'))
    #print(m.hexdigest())
    #print(m.digest())
    return m.digest()

class CFileChild():
    def __init__(self, name, pos, size):
        self.name = name
        self.pos = pos
        self.size = size

    def __str__(self):
        return self.name + '(pos:' + str(self.pos) + ', size:' + str(self.size) + ')'

class CFileList():
    def __init__(self, name, fullpath):
        self.name = name
        self.fullpath = fullpath
        self.child = []

    def addchild(self, name, pos, size):
        child = CFileChild(name, pos, size)
        self.child.append(child)

    def __str__(self):
        return self.name + '[' + ', '.join([str(x) for x in self.child]) + ']'

class CFileContainer():
    def __init__(self):
        self.log = logging.getLogger('myLogger')
        self.list = []

    def addSearchPath(self, path):
        if not path:
            return
        self.log.debug("read path:" + str(path))
        onlyfiles = [f for f in os.listdir(path) if os.path.isfile(os.path.join(path, f))]
        self.log.debug("read files:" + ','.join(onlyfiles))
        for filename in onlyfiles:
            extension = os.path.splitext(filename)[1]
            if extension == '.bnp':
                # Container for multi file
                fullpath = os.path.join(path, filename)
                size = os.path.getsize(fullpath)
                data = CFileList(filename, fullpath)
                with open(fullpath,  'rb') as fp:
                    fp.seek(size-4)
                    nOffsetFromBeginning = int.from_bytes(fp.read(4), byteorder='little', signed=False)
                    self.log.debug("[%s] nOffsetFromBeginning:%u" % (filename, nOffsetFromBeginning))
                    fp.seek(nOffsetFromBeginning)
                    nNbFile = int.from_bytes(fp.read(4), byteorder='little', signed=False)
                    self.log.debug("[%s] nNbFile:%u" % (filename, nNbFile))
                    for i in range(0, nNbFile):
                        nStringSize = int.from_bytes(fp.read(1), byteorder='little', signed=False)
                        FileName = fp.read(nStringSize).decode()
                        nFileSize2 = int.from_bytes(fp.read(4), byteorder='little', signed=False)

                        nFilePos = int.from_bytes(fp.read(4), byteorder='little', signed=False)
                        self.log.debug("[%s] (%d) sizestring:%d file:%s size2:%d pos:%d" % (filename, i, nStringSize, FileName, nFileSize2, nFilePos))
                        data.addchild(FileName, nFilePos, nFileSize2)
                    fp.close()
                self.list.append(data)

    def search(self, name):
        for x in self.list:
            for y in x.child:
                if y.name == name:
                    self.log.debug("file:%s child:%s pos:%d size:%d", x.name, y.name, y.pos, y.size)
                    return  x.fullpath, y.pos, y.size
        self.log.debug('-'*80)
        return None, None, None

    def getdata(self, name):
        fullpath, pos, size = self.search(name)
        self.log.debug("file:%s pos:%d size:%d", fullpath, pos, size)
        data = None
        with open(fullpath,  'rb') as fp:
            fp.seek(pos)
            data = fp.read(size)
            fp.close()
        return data


class TConnectionState(IntEnum):
    NotInitialised = 0      # nothing happened yet
    NotConnected = 1        # init() called
    Authenticate = 2        # connect() called, identified by the login server
    Login = 3               # connecting to the frontend, sending identification
    Synchronize = 4         # connection accepted by the frontend, synchronizing
    Connected = 5           # synchronized, connected, ready to work
    Probe = 6               # connection lost by frontend, probing for response
    Stalled = 7             # server is stalled
    Disconnect = 8          # disconnect() called, or timeout, or connection closed by frontend
    Quit = 9                # quit() called


class TActionCode(IntEnum):
    ACTION_POSITION_CODE = 0
    ACTION_GENERIC_CODE = 1
    ACTION_GENERIC_MULTI_PART_CODE = 2
    ACTION_SINT64 = 3
    ACTION_SYNC_CODE = 10
    ACTION_DISCONNECTION_CODE = 11
    ACTION_ASSOCIATION_CODE = 12
    ACTION_LOGIN_CODE = 13
    ACTION_TARGET_SLOT_CODE = 40
    ACTION_DUMMY_CODE = 99


class CLFECOMMON(IntEnum):
    SYSTEM_LOGIN_CODE = 0
    SYSTEM_SYNC_CODE = 1
    SYSTEM_ACK_SYNC_CODE = 2
    SYSTEM_PROBE_CODE = 3
    SYSTEM_ACK_PROBE_CODE = 4
    SYSTEM_DISCONNECTION_CODE = 5
    SYSTEM_STALLED_CODE = 6
    SYSTEM_SERVER_DOWN_CODE = 7
    SYSTEM_QUIT_CODE = 8
    SYSTEM_ACK_QUIT_CODE = 9
    NumBitsInLongAck = 512


class Card(IntEnum):
    BEGIN_TOKEN = 0
    END_TOKEN = 1
    SINT_TOKEN = 2
    UINT_TOKEN = 3
    FLOAT_TOKEN = 4
    STRING_TOKEN = 5
    FLAG_TOKEN = 6
    EXTEND_TOKEN = 7


class TType(IntEnum):
    STRUCT_BEGIN = 0
    STRUCT_END = 1
    FLAG = 2
    SINT32 = 3
    UINT32 = 4
    FLOAT32 = 5
    STRING = 6
    SINT64 = 7
    UINT64 = 8
    FLOAT64 = 9
    EXTEND_TYPE = 10
    NB_TYPE = 11


class TExtendType:
    ET_SHEET_ID = 0
    ET_64_BIT_EXTENDED_TYPES = 0x80000000
    ET_ENTITY_ID = 0x80000000 # ET_ENTITY_ID = ET_64_BIT_EXTENDED_TYPES


class CBNPFileVersion:
    def __init__(self):
        self.VersionNumber = None
        self.FileTime = None
        self.FileSize = None
        self.v7ZFileSize = None
        self.PatchSize = None
        self.HashKey = []

    def __str__(self):
        return "VersionNumber:" + str(self.VersionNumber) + ", FileTime:" + str(self.FileTime) + ", FileSize:" + str(self.FileSize) + ", 7ZFileSize:" + str(self.v7ZFileSize) + ", PatchSize:" + str(self.PatchSize) + ", HashKey:" + str(self.HashKey)


class CBNPFile:
    def __init__(self):
        self.FileName = None
        self.Versions = []
        self.IsIncremental = False

    def __str__(self):
        return str(self.FileName) +' (' + ', '.join( [str(x) for x in self.Versions]) + ')'

    def update(self, FileName):
        self.FileName = FileName

class CBNPCategorySet:
    def __init__(self):
        self._Name = ""
        self._IsOptional = False
        self._UnpackTo = ""
        self._IsIncremental = False
        self._CatRequired = ""
        self._Hidden = False
        self._Files = []

    def __str__(self):
        return self._Name + ' (IsOptional:' + str(self._IsOptional) + ', UnpackTo:' + self._UnpackTo + ', IsIncremental:' + str(self._IsIncremental) + ', CatRequired:' + self._CatRequired + ', Hidden:' + str(self._Hidden) + ', Files:' + str(self._Files) + ')'

# #####################################################
# persistent_data.h:140  #  struct CArg
# #####################################################
class CArgV1(Structure):
    _fields_ = [("i32_1", c_uint),
                ("i32_2", c_uint)]


class CArgV2(Structure):
    _fields_ = [("ex32_1", c_uint),
                ("ex32_2", c_uint)]


class CArgV3(Union):
    _fields_ = [("ex32", CArgV2),
                ("ExData32", c_uint),
                ("ExData64", c_ulong)]


class CArgV4(Structure):
    _fields_ = [("ExType", c_uint),
                ("ex", CArgV3)]


class CArgV5(Union):
    _fields_ = [("i", CArgV1),
                ("ii32", c_int),
                ("ii64", c_long),
                ("i32", c_uint),
                ("i64", c_ulong),
                ("f32", c_float),
                ("f64", c_double),
                ("ex", CArgV4)]

class CArg:
    #       union
    #       {
    #           struct
    #           {
    #               uint32  i32_1;
    #               uint32  i32_2;
    #           } i;
    #
    #           sint32  i32;
    #           sint64  i64;
    #           float   f32;
    #           double  f64;
    #
    #           struct
    #           {
    #               uint32  ExType;
    #               union
    #               {
    #                   struct
    #                   {
    #                       uint32  ex32_1;
    #                       uint32  ex32_2;
    #                   };
    #
    #                   uint32  ExData32;
    #                   uint64  ExData64;
    #               } ex;
    #           } ex;
    #       } _Value;

    def __init__(self):
        self._value = CArgV5()
        self._value.ii64 = 0
        self._value.i64 = 0
        self._type = 0
        self._string = 0
        self._type64 = False

    def read_Type(self):
        return self._type

    def write_Type(self, value):
        self._type = value

    def write_Type64(self, value):
        self._type64 = value

    def read_String(self):
        return self._string

    def write_String(self, value):
        self._string = value

    def read_i32_1(self):
        return self._value.i.i32_1

    def write_i32_1(self, value):
        self._value.i.i32_1 = value

    def read_i32_2(self):
        return self._value.i.i32_2

    def write_i32_2(self, value):
        self._value.i.i32_2 = value

    def read_i32(self):
        return self._value.i32

    def write_i32(self, value):
        self._value.i32 = value

    def read_i64(self):
        return self._value.i64

    def write_i64(self, value):
        self._value.i64 = value

    def read_f32(self):
        return self._value.f32

    def write_f32(self, value):
        self._value.f32 = value

    def read_f64(self):
        return self._value.f64

    def write_f64(self, value):
        self._value.f64 = value

    def read_ExType(self):
        return self._value.ex.ExType

    def write_ExType(self, value):
        self._value.ex.ExType = value

    def read_ex32_1(self):
        return self._value.ex.ex.ex32.ex32_1

    def write_ex32_1(self, value):
        self._value.ex.ex.ex32.ex32_1 = value

    def read_ex32_2(self):
        return self._value.ex.ex.ex32.ex32_2

    def write_ex32_2(self, value):
        self._value.ex.ex.ex32.ex32_2 = value

    def read_ExData32(self):
        return self._value.ex.ex.ExData32

    def write_ExData32(self, value):
        self._value.ex.ex.ExData32 = value

    def read_ExData64(self):
        return self._value.ex.ex.ExData64

    def write_ExData64(self, value):
        self._value.ex.ex.ExData64 = value

    def isExtended(self):
        if self._type == TType.EXTEND_TYPE:
            return True
        elif self._type == TType.STRUCT_BEGIN:
            self.log.error("Can't extract a value from a structure delimiter")
            sys.exit(2)
        elif self._type == TType.STRUCT_END:
            self.log.error("Can't extract a value from a structure delimiter")
            sys.exit(2)
        return False

    def isFlag(self):
        if self._type == TType.FLAG:
            return True
        else:
            return False

    def asUint(self):
        if self._type == TType.STRUCT_BEGIN or self._type == TType.STRUCT_END:
            self.log.error("Can't extract a value from a structure delimiter")
            sys.exit(2)
        elif self._type == TType.SINT32:
            return self.read_i32()
        elif self._type == TType.UINT32:
            return self.read_i32()
        elif self._type == TType.SINT64:
            return self.read_i64()
        elif self._type == TType.UINT64:
            return self.read_i64()
        elif self._type == TType.FLOAT32:
            return self.read_i32()
        elif self._type == TType.FLOAT64:
            return self.read_i64()
        elif self._type == TType.STRING:
            return int(self._string)
        elif self._type == TType.FLAG:
            return "1"
        elif self._type == TType.EXTEND_TYPE:
            if self.read_ExType() == TExtendType.ET_SHEET_ID:
                return self.read_ExData32()
            elif self.read_ExType() == TExtendType.ET_ENTITY_ID:
                return self.read_ExData64()
        log = logging.getLogger('myLogger')
        log.error("This should never happen!")
        sys.exit(2)

    def __str__(self):
        log = logging.getLogger('myLogger')
        log.debug(self._type)
        if self._type == TType.STRUCT_BEGIN or self._type == TType.STRUCT_END:
            return ''
        elif self._type64:
            # To be confirm for extend
            return str(self.read_ExData64())
        elif self._type == TType.SINT32:
            return str(self.read_i32())
        elif self._type == TType.UINT32:
            return str(self.read_i32())
        elif self._type == TType.SINT64:
            return str(self.read_i64())
        elif self._type == TType.UINT64:
            return str(self.read_i64())
        elif self._type == TType.FLOAT32:
            return str(self.read_i32())
        elif self._type == TType.FLOAT64:
            return str(self.read_i64())
        elif self._type == TType.STRING:
            return self._string
        elif self._type == TType.FLAG:
            return "1"
        return '?'

    def asSint(self):
        self.log.error("TODO")
        sys.exit(2)

    def asFloat(self):
        self.log.error("TODO")
        sys.exit(2)

    def asDouble(self):
        self.log.error("TODO")
        sys.exit(2)

    def asString(self):
        if self._type == TType.STRUCT_BEGIN or self._type == TType.STRUCT_END:
            self.log.error("Can't extract a value from a structure delimiter")
            sys.exit(2)
        elif self._type == TType.SINT32:
            return str(self.read_ii32())
        elif self._type == TType.UINT32:
            return str(self.read_i32())
        elif self._type == TType.SINT64:
            return str(self.read_ii64())
        elif self._type == TType.UINT64:
            return str(self.read_i64())
        elif self._type == TType.FLOAT32:
            return str(self.read_f32())
        elif self._type == TType.FLOAT64:
            return str(self.read_f64())
        elif self._type == TType.STRING:
            return self._string
        elif self._type == TType.FLAG:
            return "1"
        elif self._type == TType.EXTEND_TYPE:
            self.log.error("TODO")
            sys.exit(2)
            #       switch(_Value.ExType)
            #       {
            #       case ET_SHEET_ID:
            #           {
            #               NLMISC::CSheetId sheetId(_Value.ExData32);
            #               return sheetId.toString(true);
            #           }
            #       case ET_ENTITY_ID:
            #           {
            #               NLMISC::CEntityId entityId(_Value.ExData64);
            #               return entityId.toString();
            #           }
            #       default:
            #           break;
            #       }
        self.log.error("This should never happen!")
        sys.exit(2)

    def asUCString(self):
        self.log.error("TODO")
        sys.exit(2)

    def asEntityId(self):
        self.log.error("TODO")
        sys.exit(2)

    def asSheetId(self):
        self.log.error("TODO")
        sys.exit(2)

    def typeName(self):
        self.log.error("TODO")
        sys.exit(2)

# #####################################################
#
# #####################################################

class CGenericMultiPartTemp():
    def __init__(self, log):
        self.log = log
        self.NbBlock = 0xFFFFFFFF
        self.NbCurrentBlock = 0
        self.TempSize = 0
        self.Temp = []
        self.BlockReceived = []

    def set(self, Number, Part, NbBlock, PartCont):
        '''
            khanat-opennel-code/code/ryzom/client/src/network_connection.cpp # void CNetworkConnection::CGenericMultiPartTemp::set (CActionGenericMultiPart *agmp, CNetworkConnection *parent)
        '''
        if self.NbBlock == 0xFFFFFFFF:
            # Initialize
            self.NbBlock = NbBlock
            self.NbCurrentBlock = 0
            self.TempSize = 0
            self.Temp = []
            while len(self.Temp) < NbBlock:
                self.Temp.append(None)
            while len(self.BlockReceived) < NbBlock:
                self.BlockReceived.append(False)
        if self.BlockReceived[Part]:
            self.log.warning('This part is already received, discard it %d' % Part)
            return
        self.Temp[Part] = PartCont
        self.BlockReceived[Part] = True
        self.NbCurrentBlock += 1
        self.TempSize += len(PartCont)

        self.log.debug("NbCurrentBlock:%d / NbBlock:%d" % (self.NbCurrentBlock, self.NbBlock))
        if self.NbCurrentBlock == self.NbBlock:
            # reform the total action
            bms = BitStream()

            self.NbBlock == 0xFFFFFFFF
            for data in self.Temp:
                bms.pushBitStream(data)
            self.log.debug("data : %s" % bms.showAllData())
        self.log.debug("*" * 80)


class World():
    def __init__(self, log):
        self.log = log
        self.GenericMultiPartTemp = {}
        self.timestamp = 0

    def addGenericMultiPartTemp(self, id):
        self.GenericMultiPartTemp.setdefault(id, CGenericMultiPartTemp(self.log))

    def setGenericMultiPartTemp(self, Number, Part, NbBlock, PartCont):
        self.GenericMultiPartTemp[Number].set(Number, Part, NbBlock, PartCont)

class CPersistentDataRecord:
    def __init__(self,  log):
        self.log = log
        self.TokenTable = []
        self.ArgTable = []
        self.StringTable = [ ]
        self.ReadingStructStack = []
        self.offsetToken = 0
        self.ArgOffset = 0
        self.version = 0
        self.totalSize = 0
        self.tokenCount = 0
        self.argCount = 0
        self.stringCount = 0
        self.stringsSize = 0
        self.CBNPFile = []
        self.Categories = []

    def show(self):
        for x in self.CBNPFile:
            self.log.debug("File:%s" % str(x))
        for x in self.Categories:
            self.log.debug("Categorie:%s" % str(x))

    # ---------------- Manipulate Token ----------------

    # +----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+
    # |  0 |  1 |  2 |  3 |  4 |  5 |  6 |  7 |  8 |  9 | 10 | 11 | 12 | 13 | 14 | 15 |
    # +----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+
    # |   Token ID                                                     | Token Type   |
    # +----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+----+

    def token2Type(self, token, extend): # persistent_data_inline.h:1102  CPersistentDataRecord::CArg::TType CPersistentDataRecord::CArg::token2Type(uint32 token,bool extend)
        self.log.debug("token:%d, extend:%d" % (token, extend))
        if token == Card.BEGIN_TOKEN:
            return TType.STRUCT_BEGIN
        elif token == Card.END_TOKEN:
            return TType.STRUCT_END
        elif token == Card.FLAG_TOKEN:
            return TType.FLAG
        elif token == Card.SINT_TOKEN:
            if extend:
                return TType.SINT64
            else:
                return TType.SINT32
        elif token == Card.UINT_TOKEN:
            if extend:
                return TType.UINT64
            else:
                return TType.UINT32
        elif token == Card.FLOAT_TOKEN:
            if extend:
                return TType.FLOAT64
            else:
                return TType.FLOAT32
        elif token == Card.STRING_TOKEN:
            if extend:
                return TType.EXTEND_TYPE
            else:
                return TType.STRING
        self.log.error('This should never happen!')
        sys.exit(2)

    def type2Token(self, type): # persistent_data_inline.h:1118  CPersistentDataRecord::TToken CPersistentDataRecord::CArg::type2Token(uint32 type)
        self.log.debug("type: %d" %(type))
        if type == TType.STRUCT_BEGIN:
            return Card.BEGIN_TOKEN
        elif type == TType.STRUCT_END:
            return Card.END_TOKEN
        elif type == TType.FLAG:
            return Card.FLAG_TOKEN
        elif type == TType.SINT32:
            return Card.SINT_TOKEN
        elif type == TType.UINT32:
            return Card.UINT_TOKEN
        elif type == TType.FLOAT32:
            return Card.FLOAT_TOKEN
        elif type == TType.STRING:
            return Card.STRING_TOKEN
        elif type == TType.SINT64:
            return Card.SINT_TOKEN
        elif type == TType.UINT64:
            return Card.UINT_TOKEN
        elif type == TType.FLOAT64:
            return Card.FLOAT_TOKEN
        elif type == TType.EXTEND_TYPE:
            return Card.STRING_TOKEN
        self.log.error('This should never happen!')
        sys.exit(2)

    def peekNextToken(self):
        token = self.TokenTable[self.offsetToken]
        self.log.debug("[%d] token:%d" %(self.offsetToken, token))
        return token // 8 # persistent_data_inline.h:308  CPersistentDataRecord::TToken CPersistentDataRecord::peekNextToken()  # _TokenTable[_TokenOffset]>>3;

    def peekNextTokenType(self): # persistent_data_limit.h:308  CPersistentDataRecord::TToken CPersistentDataRecord::peekNextToken() const
        self.log.debug("peekNextTokenType - old offset token:%d" % self.offsetToken)
        if self.isEndOfData():
            self.log.error('Attempt to read past end of input data')
            sys.exit(2)
        token = self.TokenTable[self.offsetToken]
        tokenType = token & 7
        if tokenType == Card.EXTEND_TOKEN:
            if self.offsetToken + 1 > self.tokenCount:
                self.log.error('Attempt to read past end of input data')
                sys.exit(2)
            tokenType = self.TokenTable[self.offsetToken+1]
            self.log.debug("peekNextTokenType [%d] token:%d type:%d" %(self.offsetToken, token, tokenType))
            return self.token2Type(tokenType, True)
        self.log.debug("peekNextTokenType [%d] token:%d type:%d" %(self.offsetToken, token, tokenType))
        return self.token2Type(tokenType, False)

    def isEndOfData(self):
        if self.offsetToken == self.tokenCount:
            return True
        return False

    def isEndOfStruct(self):
        if self.isEndOfData():
            self.log.debug("isEndOfData")
            return True
        elif len(self.ReadingStructStack) == 0:
            self.log.debug("ReadingStructStack")
            return False
        elif self.peekNextTokenType() != TType.STRUCT_END:
            self.log.debug("peekNextTokenType != TType.STRUCT_END")
            return False
        elif self.ReadingStructStack[-1] != self.peekNextToken():
            self.log.error("Opening and closing structure tokens don't match")
            sys.exit(2)
        self.log.debug("isEndOfStruct")
        return True

    def isStartOfStruct(self):
        if self.peekNextTokenType() == TType.STRUCT_BEGIN:
            return True
        return False

    def popStructBegin(self, token):
        if self.peekNextToken() != token:
            self.log.error('Attempting to enter a structure with the wrong delimiting token')
            sys.exit(2)
        if self.peekNextTokenType() != TType.STRUCT_BEGIN:
            self.log.error('Attempting to leave a structure with the wrong delimiting token type')
            sys.exit(2)
        self.ReadingStructStack.append(token)
        self.offsetToken += 1

    def popStructEnd(self, token):
        if len(self.ReadingStructStack) == 0:
            self.log.error('Attempting to pop end of a structure with nothing left in the open structure stack')
            sys.exit(2)
        nextToken = self.peekNextToken()
        topToken = self.ReadingStructStack[-1]
        if topToken != token:
            self.log.error('Attempting to pop end of a structure with the wrong delimiting token')
            sys.exit(2)
        if nextToken != token:
            self.log.error('Attempting to pop end of a structure with the wrong delimiting token')
            sys.exit(2)
        if self.peekNextTokenType() != TType.STRUCT_END:
            self.log.error('Attempting to leave a structure with the wrong delimiting token type')
            sys.exit(2)
        del self.ReadingStructStack[-1]
        self.offsetToken += 1

    # ---------------- Manipulate StringTable ----------------
    def lookupString(self, idx):
        if idx >= self.stringCount:
            self.log.error("Attempting to access past end of string table")
            sys.exit(2)
        return self.StringTable[idx]

    # ---------------- Manipulate Arg ----------------
    def peekNextArg(self): # persistent_data_limit.h:339  CPersistentDataRecord::peekNextArg(CPersistentDataRecord::CArg& result) const
        _type = self.peekNextTokenType()
        result = CArg()
        result.write_Type(_type)
        result.write_Type64(False)
        self.log.debug("peekNextArg - Type:%d ArgOffset:%d" % (_type, self.ArgOffset))
        if result.isExtended():
            self.log.debug("Extended")
            result.write_i32_1(self.ArgTable[self.ArgOffset])
            result.write_i32_2(self.ArgTable[self.ArgOffset+1])
            if result.read_Type() == TType.EXTEND_TYPE and result.read_ExType() == TExtendType.ET_64_BIT_EXTENDED_TYPES:
                result.write_ex32_2(self.ArgTable[self.ArgOffset+2]);
                result.write_Type64(True)
        elif not result.isFlag():
            # result._Value.i32_1 = _ArgTable[_ArgOffset];
            result.write_i32_1(self.ArgTable[self.ArgOffset])
            self.log.debug("peekNextArg - id :%d" % result.read_i32_1())
        if result.read_Type() == TType.STRING:
            result.write_String(self.lookupString(result.read_i32_1()))
            self.log.debug("peekNextArg - String:%s" % result.read_String())
        return result

    def popNextArg(self, token): # persistent_data_limit.h:414  CPersistentDataRecord::popNextArg(TToken token,CPersistentDataRecord::CArg& result)
        result = self.peekNextArg()
        if result.isFlag():
            self.offsetToken += 1
        elif result.isExtended():
            self.ArgOffset += 2
            self.offsetToken += 2
            if result.read_Type() == TType.EXTEND_TYPE and result.read_ExType() == TExtendType.ET_64_BIT_EXTENDED_TYPES:
                self.ArgOffset += 1
                self.offsetToken += 1
        else:
            self.ArgOffset += 1
            self.offsetToken += 1
        self.log.debug("popNextArg - Arg:%d", result.read_i32_1())
        return result

    def popString(self, token):
        TempArg = self.popNextArg(token)
        return TempArg.asString()

    def popUint32(self, token):
        TempArg = self.popNextArg(token)
        return TempArg.asUint()

    def popBool(self, token):
        TempArg = self.popNextArg(token)
        return TempArg.asUint() != 0

    # ---------------- Read Data ----------------
    def readFromBinFile(self, filename): # persistent_data.cpp:835   # bool CPersistentDataRecord::fromBuffer(const char *src, uint32 bufferSize)
        self.log.debug('Read Bin File %s' % filename)
        with open(filename, "rb") as fp:
            buffer = fp.read()
            fp.close()

            self.version = int.from_bytes(buffer[0:4], byteorder='little', signed=False)
            self.totalSize = int.from_bytes(buffer[4:8], byteorder='little', signed=False)
            self.tokenCount = int.from_bytes(buffer[8:12], byteorder='little', signed=False)
            self.argCount = int.from_bytes(buffer[12:16], byteorder='little', signed=False)
            self.stringCount = int.from_bytes(buffer[16:20], byteorder='little', signed=False)
            self.stringsSize = int.from_bytes(buffer[20:24], byteorder='little', signed=False)
            offset = 24
            self.log.debug("version:%d, totalSize:%d, tokenCount:%d, argCount:%d, stringCount:%d, stringsSize:%d" % (self.version, self.totalSize, self.tokenCount, self.argCount, self.stringCount, self.stringsSize))
            if  len(buffer) != self.totalSize:
                self.log.error("Failed to parse buffer due to invalid header (file:%s, size:%d, size define:%d)" % (filename, len(buffer), self.totalSize ))
                sys.exit(2)
            if self.version > 0:
                self.log.error("PDR ERROR: Wrong file format version! (file:%s, version:%d)" % (filename, self.version))
                sys.exit(2)
            if (self.stringCount != 0 and self.stringsSize == 0) or (self.stringCount == 0 and self.stringsSize != 0):
                self.log.error("PDR ERROR: Invalid string table parameters! (file:%s, stringCount:%d, stringsSize:%d)" % (filename, self.stringCount, self.stringsSize))
                sys.exit(2)
            # i = offset+tokenCount*sizeof(TToken)+argCount*sizeof(uint32)+stringsSize
            i = offset + self.tokenCount * 2 + self.argCount * 4 + self.stringsSize;
            if self.totalSize != i:
                self.log.error("PDR ERROR: Invalid source data (file:%s, totalSize:%d != datasize:%s)" % (filename, self.totalSize, i))
                sys.exit(2)

            # READ the tokens
            self.TokenTable = []
            for i in range(0, self.tokenCount):
                tmp = int.from_bytes(buffer[offset:offset+2], byteorder='little', signed=False)
                self.log.debug("token %5d => %3d id:%3d type:%d" %(i,  tmp, tmp // 8, tmp & 7))
                self.TokenTable.append(tmp)
                offset += 2

            # READ the arguments
            self.ArgTable = []
            for i in range(0, self.argCount):
                tmp = int.from_bytes(buffer[offset:offset+4], byteorder='little', signed=False)
                self.ArgTable.append(tmp)
                offset += 4

            # READ the string table data
            if self.stringsSize != 0:
                chaine = ''
                self.StringTable = [ ]
                while offset < self.totalSize:
                    car = buffer[offset:offset+1].decode()
                    if car != '\0':
                        chaine += car
                    else:
                        self.StringTable.append(chaine)
                        chaine = ''
                    offset += 1
                self.log.debug(self.StringTable)
                if chaine != '':
                    self.log.error("PDR ERROR: Too few strings found in string table (file:%s)" % (filename))
                    sys.exit(2)
            self.log.debug("Red %s" % filename)

    def decrypt_token(self):
        i = 0
        lvl = 0
        posArg = 0
        extend = False
        extend64 = False
        result = CArg()

        print("^  Position  ^  Token  ^")
        for value in self.TokenTable:
            print("|  %5d |  %3d |" %(i, value))
            i += 1

        i = 0
        print("^  Position  ^  Argument  ^")
        for value in self.ArgTable:
            print("|  %5d |  %3d |" %(i, value))
            i += 1

        i = 0
        print("^  Position  ^  String  ^")
        for value in self.StringTable:
            print("|  %5d |  %s |" %(i, value))
            i += 1

        i = 0
        print("^  Position  ^  Niveau  ^  Token  ^  Token ID  ^^  Token Type (Card)  ^^^ Result ^")
        print("^ ^^ (entrée) ^ Valeur ^ Quoi ^ Valeur ^  Card ^  Type ^ ^")
        for token in self.TokenTable:
            tokenId = token // 8
            tokenTypeValue = token & 7
            result.write_String("-")
            if tokenTypeValue == 0:
                tokenCard = 'BEGIN_TOKEN'
                tokenType =  'STRUCT_BEGIN'
                result.write_Type(TType.STRUCT_BEGIN)
                if lvl <= 1:
                    print("| |||||||")
                lvl += 1
            elif tokenTypeValue == 1:
                tokenCard = 'END_TOKEN'
                tokenType =  'STRUCT_END'
                result.write_Type(TType.STRUCT_END)
                extend = False
                extend64 = False
            elif tokenTypeValue == 2:
                tokenCard = 'SINT_TOKEN'
                if extend:
                    tokenType =  'SINT64'
                    result.write_Type(TType.SINT64)
                    result.write_i32_1(self.ArgTable[posArg])
                    result.write_i32_2(self.ArgTable[posArg+1])
                    if extend64:
                        result.write_ex32_2(self.ArgTable[posArg+2]);
                        posArg += 3
                    else:
                        posArg += 2
                else:
                    tokenType =  'SINT32'
                    result.write_Type(TType.SINT32)
                    result.write_i32_1(self.ArgTable[posArg])
                    posArg += 1
                extend = False
                extend64 = False
            elif tokenTypeValue == 3:
                tokenCard = 'UINT_TOKEN'
                if extend:
                    tokenType =  'UINT64'
                    result.write_Type(TType.UINT64)
                    result.write_i32_1(self.ArgTable[posArg])
                    result.write_i32_2(self.ArgTable[posArg+1])
                    if extend64:
                        result.write_ex32_2(self.ArgTable[posArg+2]);
                        posArg += 3
                    else:
                        posArg += 2
                else:
                    tokenType =  'UINT32'
                    result.write_Type(TType.UINT32)
                    result.write_i32_1(self.ArgTable[posArg])
                    posArg += 1
                extend = False
                extend64 = False
            elif tokenTypeValue == 4:
                tokenCard = 'FLOAT_TOKEN'
                if extend:
                    tokenType =  'FLOAT64'
                    result.write_Type(TType.FLOAT64)
                    result.write_i32_1(self.ArgTable[posArg])
                    result.write_i32_2(self.ArgTable[posArg+1])
                    if extend64:
                        result.write_ex32_2(self.ArgTable[posArg+2]);
                        posArg += 3
                    else:
                        posArg += 2
                else:
                    tokenType =  'FLOAT32'
                    result.write_Type(TType.FLOAT32)
                    result.write_i32_1(self.ArgTable[posArg])
                    posArg += 1
                extend = False
                extend64 = False
            elif tokenTypeValue == 5:
                tokenCard = 'STRING_TOKEN'
                if extend:
                    tokenType =  'EXTEND_TYPE'
                    result.write_Type(TType.EXTEND_TYPE)
                    result.write_i32_1(self.ArgTable[posArg])
                    result.write_i32_2(self.ArgTable[posArg+1])
                    if extend64:
                        result.write_ex32_2(self.ArgTable[posArg+2]);
                        posArg += 3
                    else:
                        posArg += 2
                else:
                    tokenType =  'STRING'
                    result.write_Type(TType.STRING)
                    result.write_i32_1(self.ArgTable[posArg])
                    tmp = result.read_i32_1()
                    result.write_String(self.StringTable[tmp])
                    posArg += 1
                extend = False
                extend64 = False
            elif tokenType == 6:
                tokenCard = 'FLAG_TOKEN'
                tokenType =  'FLAG'
                result.write_Type(TType.FLAG)
                extend = False
                extend64 = False
            elif tokenTypeValue == 7:
                if extend:
                    extend64 = True
                tokenCard = 'EXTEND_TOKEN'
                result.write_Type(TType.EXTEND_TYPE)
                tokenType =  ''
                extend = True
            # print("token %5d => %3d id:%3d [%s] type:%d [%s]" %(i, token, tokenId, self.StringTable[tokenId], tokenType, tokenCard))
            print("|  %5d |  %3d |  %3d |  %3d | %s |  %d | %s | %s | %s |" %(i, lvl,  token, tokenId, self.StringTable[tokenId], tokenTypeValue, tokenCard , tokenType, result))
            if tokenTypeValue == 1:
                lvl -= 1
            i += 1

    def addString(self, name): # persistent_data.cpp:100  uint16 CPersistentDataRecord::addString(const string& name)
        for i in range(0, len(self.StringTable)):
            if self.StringTable[i] == name:
                return i
        self.StringTable.append(name)
        return len(self.StringTable) - 1

    def CProductDescriptionForClient_apply(self): # persistent_data_template.h:459  # void PERSISTENT_CLASS::apply(CPersistentDataRecord &pdr _PERSISTENT_APPLY_ARGS)
        __Tok__MapKey = self.addString("__Key__")
        __Tok__MapVal = self.addString("__Val__")
        __Tok_Files = self.addString("_Files")
        __Tok_Categories = self.addString("_Categories")
        self.log.debug("MapKey:%d, MapVal:%d, Files:%d, Categories:%d" %(__Tok__MapKey, __Tok__MapVal, __Tok_Files, __Tok_Categories))
        while not self.isEndOfStruct():
            nextToken = self.peekNextToken()
            self.log.debug("nextToken:%d" % (nextToken))
            if nextToken == __Tok_Files:
                self.popStructBegin(__Tok_Files)
                self.CBNPFileSet_apply()
                self.popStructEnd(__Tok_Files)
                continue
            elif nextToken == __Tok_Categories:
                self.popStructBegin(__Tok_Categories)
                # (_Categories).apply(pdr);
                self.CBNPCategorySet_apply()
                self.popStructEnd(__Tok_Categories)
                continue
            self.log.error("TODO")
            sys.exit(2)

    def CBNPFileSet_apply(self):
        __Tok__MapKey = self.addString("__Key__")
        __Tok__MapVal = self.addString("__Val__")
        __Tok_Files = self.addString("_Files")
        self.log.debug("MapKey:%d, MapVal:%d, Files:%d" %(__Tok__MapKey, __Tok__MapVal, __Tok_Files))
        while not self.isEndOfStruct():
            nextToken = self.peekNextToken()
            self.log.debug("nextToken:%d" % (nextToken))
            if nextToken == __Tok_Files:
                self.popStructBegin(__Tok_Files)
                self.CBNPFile.append(CBNPFile())
                self.CBNPFile_apply(self.CBNPFile[-1])
                self.popStructEnd(__Tok_Files)
                continue
            self.log.error("TODO")
            sys.exit(2)

    def CBNPFile_apply(self, _CBNPFile):
        __Tok__MapKey = self.addString("__Key__")
        __Tok__MapVal = self.addString("__Val__")
        __Tok_FileName = self.addString("_FileName")
        __Tok_Versions = self.addString("_Versions")
        _FileName = None
        self.log.debug("MapKey:%d, MapVal:%d, Filename:%d, Versions:%d" %(__Tok__MapKey, __Tok__MapVal, __Tok_FileName, __Tok_Versions))
        while not self.isEndOfStruct():
            nextToken = self.peekNextToken()
            self.log.debug("nextToken:%d" % (nextToken))
            if nextToken == __Tok_FileName:
                _FileName = self.popString(nextToken)
                _CBNPFile.FileName = _FileName
                self.log.debug("filename: %s" % _FileName)
                continue
            if nextToken == __Tok_Versions:
                self.popStructBegin(__Tok_Versions)
                # vectAppend(_Versions).apply(pdr);
                _CBNPFile.Versions.append(CBNPFileVersion())
                self.CBNPFileVersion_apply(_CBNPFile.Versions[-1])
                self.popStructEnd(__Tok_Versions)
                continue
            stack = []
            while True:
                if self.isStartOfStruct():
                    stack.append(self.peekNextToken())
                    self.popStructBegin(stack)
                elif self.isEndOfStruct():
                    self.popStructEnd(stack[-1])
                    if len(stack) > 0:
                        del stack[-1]
                else:
                    self.popNextArg(self.peekNextToken())
                if self.isEndOfData() and len(stack) == 0:
                    break
        self.log.debug("CBNPFile: %s" % _CBNPFile)

    def CBNPFileVersion_apply(self, _CBNPFileVersion): # persistent_data_template.h:459 # void CBNPFileVersion::apply(CPersistentDataRecord &pdr )
        __Tok__MapKey = self.addString("__Key__")
        __Tok__MapVal = self.addString("__Val__")
        __Tok_VersionNumber = self.addString("_VersionNumber")
        __Tok_FileSize = self.addString("_FileSize")
        __Tok_7ZFileSize = self.addString("_7ZFileSize")
        __Tok_FileTime = self.addString("_FileTime")
        __Tok_PatchSize = self.addString("_PatchSize")
        __Tok_HashKey = self.addString("_HashKey")
        self.log.debug("MapKey:%d, MapVal:%d, VersionNumber:%d, FileSize:%d, 7ZFileSize:%d, FileTime:%d, PatchSize:%d, HashKey:%d" %(__Tok__MapKey, __Tok__MapVal, __Tok_VersionNumber, __Tok_FileSize, __Tok_7ZFileSize, __Tok_FileTime, __Tok_PatchSize, __Tok_HashKey))

        while not self.isEndOfStruct():
            nextToken = self.peekNextToken()
            self.log.debug("nextToken:%d" % (nextToken))
            if nextToken == __Tok_VersionNumber:
                self.log.debug("__Tok_VersionNumber")
                _CBNPFileVersion.VersionNumber = self.popUint32(__Tok_VersionNumber)
                self.log.debug("VersionNumber: %s" % _CBNPFileVersion.VersionNumber)
                continue
            elif nextToken == __Tok_FileSize:
                self.log.debug("__Tok_FileSize")
                _CBNPFileVersion.FileSize = self.popUint32(__Tok_FileSize)
                self.log.debug("FileSize: %s" % _CBNPFileVersion.FileSize)
                continue
            elif nextToken == __Tok_7ZFileSize:
                self.log.debug("__Tok_7ZFileSize")
                _CBNPFileVersion.v7ZFileSize = self.popUint32(__Tok_7ZFileSize)
                self.log.debug("7ZFileSize: %s" % _CBNPFileVersion.v7ZFileSize)
                continue
            elif nextToken == __Tok_FileTime:
                self.log.debug("__Tok_FileTime")
                _CBNPFileVersion.FileTime = self.popUint32(__Tok_FileTime)
                self.log.debug("FileTime: %s" % _CBNPFileVersion.FileTime)
                continue
            elif nextToken == __Tok_PatchSize:
                self.log.debug("__Tok_PatchSize")
                _CBNPFileVersion.PatchSize = self.popUint32(__Tok_PatchSize)
                self.log.debug("PatchSize: %s" % _CBNPFileVersion.PatchSize)
                continue
            elif nextToken == __Tok_HashKey:
                self.log.debug("__Tok_HashKey")
                _CBNPFileVersion.HashKey.append(self.popUint32(__Tok_HashKey))
                self.log.debug("HashKey: %s" % _CBNPFileVersion.HashKey[-1])
                continue
            # Vidage des autres clefs (inconnues)
            stack = []
            while True:
                if self.isStartOfStruct():
                    stack.append(self.peekNextToken())
                    self.popStructBegin(stack)
                elif self.isEndOfStruct():
                    self.popStructEnd(stack[-1])
                    if len(stack) > 0:
                        del stack[-1]
                else:
                    self.popNextArg(self.peekNextToken())
                if self.isEndOfData() and len(stack) == 0:
                    break

    def CBNPCategorySet_apply(self): # persistent_data_template.h:459 # void CBNPCategorySet::apply(CPersistentDataRecord &pdr )
        #__Tok__MapKey = self.addString("__Key__")
        #__Tok__MapVal = self.addString("__Val__")
        __Tok_Category = self.addString("_Category")
        while not self.isEndOfStruct():
            nextToken = self.peekNextToken()
            self.log.debug("nextToken:%d" % (nextToken))
            if nextToken == __Tok_Category:
                self.log.debug("__Tok_Category")
                self.popStructBegin(__Tok_Category)
                self.Categories.append(CBNPCategorySet())
                self.CBNPCategory_apply(self.Categories[-1])
                self.popStructEnd(__Tok_Category)
                continue
            # Vidage des autres clefs (inconnues)
            stack = []
            while True:
                if self.isStartOfStruct():
                    stack.append(self.peekNextToken())
                    self.popStructBegin(stack)
                elif self.isEndOfStruct():
                    self.popStructEnd(stack[-1])
                    if len(stack) > 0:
                        del stack[-1]
                else:
                    self.popNextArg(self.peekNextToken())
                if self.isEndOfData() and len(stack) == 0:
                    break


    def CBNPCategory_apply(self, _CBNPCategory): # persistent_data_template.h:459 # void CBNPCategory::apply(CPersistentDataRecord &pdr )
        __Tok__MapKey = self.addString("__Key__")
        __Tok__MapVal = self.addString("__Val__")
        __Tok_Name = self.addString("_Name")
        __Tok_IsOptional = self.addString("_IsOptional")
        __Tok_UnpackTo = self.addString("_UnpackTo")
        __Tok_IsIncremental = self.addString("_IsIncremental")
        __Tok_CatRequired = self.addString("_CatRequired")
        __Tok_Hidden = self.addString("_Hidden")
        __Tok_Files = self.addString("_Files")
        self.log.debug("MapKey:%d, MapVal:%d, Name:%d, IsOptional:%d, UnpackTo:%d, IsIncremental:%d, CatRequired:%d, Hidden:%d, Files:%d" %(__Tok__MapKey, __Tok__MapVal, __Tok_Name, __Tok_IsOptional, __Tok_UnpackTo, __Tok_IsIncremental, __Tok_CatRequired, __Tok_Hidden, __Tok_Files))
        while not self.isEndOfStruct():
            nextToken = self.peekNextToken()
            self.log.debug("nextToken:%d" % (nextToken))
            if nextToken == __Tok_Name:
                self.log.debug("__Tok_Name")
                _CBNPCategory._Name = self.popString(nextToken)
                self.log.debug("_Name: %s" % _CBNPCategory._Name)
                continue
            elif nextToken == __Tok_IsOptional:
                self.log.debug("__Tok_IsOptional")
                _CBNPCategory._IsOptional = self.popBool(nextToken)
                self.log.debug("_IsOptional: %s" % str(_CBNPCategory._IsOptional))
                continue
            elif nextToken == __Tok_UnpackTo:
                self.log.debug("__Tok_UnpackTo")
                _CBNPCategory._UnpackTo = self.popString(nextToken)
                self.log.debug("_UnpackTo: %s" % str(_CBNPCategory._UnpackTo))
                continue
            elif nextToken == __Tok_IsIncremental:
                self.log.debug("__Tok_IsIncremental")
                _CBNPCategory._IsIncremental = self.popBool(nextToken)
                self.log.debug("_IsIncremental: %s" % str(_CBNPCategory._IsIncremental))
                continue
            elif nextToken == __Tok_CatRequired:
                self.log.debug("__Tok_CatRequired")
                _CBNPCategory._CatRequired = self.popString(nextToken)
                self.log.debug("_CatRequired: %s" % str(_CBNPCategory._CatRequired))
                continue
            elif nextToken == __Tok_Hidden:
                self.log.debug("__Tok_Hidden")
                _CBNPCategory._Hidden = self.popBool(nextToken)
                self.log.debug("_Hidden: %s" % str(_CBNPCategory._Hidden))
                continue
            elif nextToken == __Tok_Files:
                self.log.debug("__Tok_Files")
                _CBNPCategory._Files.append(self.popString(nextToken))
                self.log.debug("_Files: %s" % str(_CBNPCategory._Files))
                continue
            # Vidage des autres clefs (inconnues)
            stack = []
            while True:
                if self.isStartOfStruct():
                    stack.append(self.peekNextToken())
                    self.popStructBegin(stack)
                elif self.isEndOfStruct():
                    self.popStructEnd(stack[-1])
                    if len(stack) > 0:
                        del stack[-1]
                else:
                    self.popNextArg(self.peekNextToken())
                if self.isEndOfData() and len(stack) == 0:
                    break


def getPowerOf2(v):
    res=1;
    ret=0;
    while res<v:
        ret += 1
        res *= 2
    return ret

class DecodeImpulse():
    def __init__(self, log, world):
        '''
            khanat-opennel-code/code/ryzom/client/src/net_manager.cpp # void initializeNetwork()
        '''
        self.log = log
        self.msgXml = None
        self.databaseXml = None
        self.GenericMsgHeaderMngr = {}
        self.initializeNetwork()
        self.world = world

    def impulseDatabaseUpdatePlayer(self, msgin):
        self.log.debug("TODO:%s" % msgin)
    def impulseUpdateInventory(self, msgin):
        self.log.debug("TODO")
    def impulseInitInventory(self, msgin):
        self.log.debug("TODO")
    def impulseDatabaseUpdateBank(self, msgin):
        self.log.debug("TODO")
    def impulseDatabaseInitBank(self, msgin):
        self.log.debug("TODO")
    def impulseDatabaseResetBank(self, msgin):
        self.log.debug("TODO")
    def impulseNoUserChar(self, msgin):
        self.log.debug("TODO")
    def impulseUserChars(self, msgin):
        self.log.debug("TODO")
    def impulseUserChar(self, msgin):
        self.log.debug("TODO")
    def impulseFarTP(self, msgin):
        self.log.debug("TODO")
    def impulseServerReady(self, msgin):
        self.log.debug("TODO")
    def impulseCharNameValid(self, msgin):
        self.log.debug("TODO")
    def impulseShardId(self, msgin):
        self.log.debug("TODO")
    def impulseServerQuitOk(self, msgin):
        self.log.debug("TODO")
    def impulseServerQuitAbort(self, msgin):
        self.log.debug("TODO")
    def impulseMailNotification(self, msgin):
        self.log.debug("TODO")
    def impulseForumNotification(self, msgin):
        self.log.debug("TODO")
    def impulsePermanentBan(self, msgin):
        self.log.debug("TODO")
    def impulsePermanentUnban(self, msgin):
        self.log.debug("TODO")

    def impulseChat(self, msgin):
        self.log.debug("TODO")
    def impulseTell(self, msgin):
        self.log.debug("TODO")
    def impulseFarTell(self, msgin):
        self.log.debug("TODO")
    def impulseChat2(self, msgin):
        self.log.debug("TODO")
    def impulseDynString(self, msgin):
        self.log.debug("TODO")
    def inpulseDynStringInChatGroup(self, msgin):
        self.log.debug("TODO")
    def impulseTell2(self, msgin):
        self.log.debug("TODO")

    def impulseTP(self, msgin):
        self.log.debug("TODO")
    def impulseTPWithSeason(self, msgin):
        self.log.debug("TODO")
    def impulseCorrectPos(self, msgin):
        self.log.debug("TODO")
    def impulseCombatEngageFailed(self, msgin):
        self.log.debug("TODO")
    def impulseDynChatOpen(self, msgin):
        self.log.debug("TODO")
    def impulseDynChatClose(self, msgin):
        self.log.debug("TODO")

    def impulseBeginCast(self, msgin):
        self.log.debug("TODO")
    def impulseTeamInvitation(self, msgin):
        self.log.debug("TODO")
    def impulseTeamShareOpen(self, msgin):
        self.log.debug("TODO")
    def impulseTeamShareInvalid(self, msgin):
        self.log.debug("TODO")
    def impulseTeamShareClose(self, msgin):
        self.log.debug("TODO")
    def impulseTeamContactInit(self, msgin):
        self.log.debug("TODO")
    def impulseTeamContactCreate(self, msgin):
        self.log.debug("TODO")
    def impulseTeamContactStatus(self, msgin):
        self.log.debug("TODO")
    def impulseTeamContactRemove(self, msgin):
        self.log.debug("TODO")

    def impulseExchangeInvitation(self, msgin):
        self.log.debug("TODO")
    def impulseExchangeCloseInvitation(self, msgin):
        self.log.debug("TODO")
    def impulseMountAbort(self, msgin):
        self.log.debug("TODO")

    def impulseWhere(self, msgin):
        self.log.debug("TODO")
    def impulseCounter(self, msgin):
        self.log.debug("TODO")


    def impulsePhraseSend(self, msgin):
        self.log.debug("TODO")

    def impulseStringResp(self, msgin):
        self.log.debug("TODO")
    def impulseReloadCache(self, msgin):
        self.world.timestamp = msgin.readUint32('timestamp')
        self.log.debug("Reload Cache timestamp:%d" % self.world.timestamp)
        self.log.debug("Message not read (%d) %s" % (msgin.needRead(), msgin.showLastData() ))

    def impulseBotChatForceEnd(self, msgin):
        self.log.debug("TODO")

    def impulseJournalInitCompletedMissions(self, msgin):
        self.log.debug("TODO")
    def impulseJournalUpdateCompletedMissions(self, msgin):
        self.log.debug("TODO")


    def impulseJournalAddCompass(self, msgin):
        self.log.debug("TODO")
    def impulseJournalRemoveCompass(self, msgin):
        self.log.debug("TODO")

    def impulseGuildJoinProposal(self, msgin):
        self.log.debug("TODO")

    def impulseGuildAscensor(self, msgin):
        self.log.debug("TODO")
    def impulseGuildLeaveAscensor(self, msgin):
        self.log.debug("TODO")
    def impulseGuildAbortCreation(self, msgin):
        self.log.debug("TODO")
    def impulseGuildOpenGuildWindow(self, msgin):
        self.log.debug("TODO")

    def impulseGuildOpenInventory(self, msgin):
        self.log.debug("TODO")
    def impulseGuildCloseInventory(self, msgin):
        self.log.debug("TODO")

    def impulseGuildUpdatePlayerTitle(self, msgin):
        self.log.debug("TODO")
    def impulseGuildUseFemaleTitles(self, msgin):
        self.log.debug("TODO")


    def impulseCloseTempInv(self, msgin):
        self.log.debug("TODO")

    def impulseRemoteAdmin(self, msgin):
        self.log.debug("TODO")

    def impulsePhraseDownLoad(self, msgin):
        self.log.debug("TODO")
    def impulsePhraseConfirmBuy(self, msgin):
        self.log.debug("TODO")
    def impulsePhraseAckExecuteCyclic(self, msgin):
        self.log.debug("TODO")
    def impulsePhraseAckExecuteNext(self, msgin):
        self.log.debug("TODO")

    def impulseItemInfoSet(self, msgin):
        self.log.debug("TODO")
    def impulseItemInfoRefreshVersion(self, msgin):
        self.log.debug("TODO")
    def impulsePrereqInfoSet(self, msgin):
        self.log.debug("TODO")
    def impulseItemOpenRoomInventory(self, msgin):
        self.log.debug("TODO")
    def impulseItemCloseRoomInventory(self, msgin):
        self.log.debug("TODO")

    def impulseDeathRespawnPoint(self, msgin):
        self.log.debug("TODO")
    def impulseDeathRespawn(self, msgin):
        self.log.debug("TODO")

    def impulseDuelInvitation(self, msgin):
        self.log.debug("TODO")
    def impulseDuelCancelInvitation(self, msgin):
        self.log.debug("TODO")

    def impulsePVPChallengeInvitation(self, msgin):
        self.log.debug("TODO")
    def impulsePVPChallengeCancelInvitation(self, msgin):
        self.log.debug("TODO")

    def impulsePVPFactionPushFactionWar(self, msgin):
        self.log.debug("TODO")
    def impulsePVPFactionPopFactionWar(self, msgin):
        self.log.debug("TODO")
    def impulsePVPFactionFactionWars(self, msgin):
        self.log.debug("TODO")

    def impulseEncyclopediaUpdate(self, msgin):
        self.log.debug("TODO")
    def impulseEncyclopediaInit(self, msgin):
        self.log.debug("TODO")

    def impulseUserBars(self, msgin):
        self.log.debug("TODO")
    def impulseUserPopup(self, msgin):
        self.log.debug("TODO")


    def impulseEnterCrZoneProposal(self, msgin):
        self.log.debug("TODO")
    def impulseCloseEnterCrZoneProposal(self, msgin):
        self.log.debug("TODO")


    def     cbImpulsionGatewayOpen(self, msgin):
        self.log.debug("TODO")
    def cbImpulsionGatewayMessage (self, msgin):
        self.log.debug("TODO")
    def     cbImpulsionGatewayClose (self, msgin):
        self.log.debug("TODO")

    def impulseOutpostChooseSide (self, msgin):
        self.log.debug("TODO")
    def impulseOutpostDeclareWarAck (self, msgin):
        self.log.debug("TODO")

    def impulseCombatFlyingChaScore1Delta (self, msgin):
        self.log.debug("TODO")
    def impulseCombatFlyingTextItemSpecialEffectProc (self, msgin):
        self.log.debug("TODO")
    def impulseCombatFlyingText (self, msgin):
        self.log.debug("TODO")

    def impulseSetSeason (self, msgin):
        self.log.debug("TODO")
    def impulseDssDown (self, msgin):
        self.log.debug("TODO")

    def impulseSetNpcIconDesc (self, msgin):
        self.log.debug("TODO")
    def impulseServerEventForMissionAvailability (self, msgin):
        self.log.debug("TODO")
    def impulseSetNpcIconTimer (self, msgin):
        self.log.debug("TODO")

    def impulseDatabaseInitPlayer(self, msgin):
        self.log.debug("TODO")

    def initializeNetwork(self):
        self.GenericMsgHeaderMngr.setdefault('DB_UPD_PLR', self.impulseDatabaseUpdatePlayer)
        self.GenericMsgHeaderMngr.setdefault('DB_INIT:PLR', self.impulseDatabaseInitPlayer)
        self.GenericMsgHeaderMngr.setdefault("DB_UPD_INV", self.impulseUpdateInventory)
        self.GenericMsgHeaderMngr.setdefault("DB_INIT:INV", self.impulseInitInventory)
        self.GenericMsgHeaderMngr.setdefault("DB_GROUP:UPDATE_BANK", self.impulseDatabaseUpdateBank)
        self.GenericMsgHeaderMngr.setdefault("DB_GROUP:INIT_BANK", self.impulseDatabaseInitBank)
        self.GenericMsgHeaderMngr.setdefault("DB_GROUP:RESET_BANK", self.impulseDatabaseResetBank)
        self.GenericMsgHeaderMngr.setdefault("CONNECTION:NO_USER_CHAR", self.impulseNoUserChar)
        self.GenericMsgHeaderMngr.setdefault("CONNECTION:USER_CHARS", self.impulseUserChars)
        self.GenericMsgHeaderMngr.setdefault("CONNECTION:USER_CHAR", self.impulseUserChar)
        self.GenericMsgHeaderMngr.setdefault("CONNECTION:FAR_TP", self.impulseFarTP)
        self.GenericMsgHeaderMngr.setdefault("CONNECTION:READY", self.impulseServerReady)
        self.GenericMsgHeaderMngr.setdefault("CONNECTION:VALID_NAME", self.impulseCharNameValid)
        self.GenericMsgHeaderMngr.setdefault("CONNECTION:SHARD_ID", self.impulseShardId)
        self.GenericMsgHeaderMngr.setdefault("CONNECTION:SERVER_QUIT_OK", self.impulseServerQuitOk)
        self.GenericMsgHeaderMngr.setdefault("CONNECTION:SERVER_QUIT_ABORT", self.impulseServerQuitAbort)
        self.GenericMsgHeaderMngr.setdefault("CONNECTION:MAIL_AVAILABLE", self.impulseMailNotification)
        self.GenericMsgHeaderMngr.setdefault("CONNECTION:GUILD_MESSAGE_AVAILABLE", self.impulseForumNotification)
        self.GenericMsgHeaderMngr.setdefault("CONNECTION:PERMANENT_BAN", self.impulsePermanentBan)
        self.GenericMsgHeaderMngr.setdefault("CONNECTION:UNBAN", self.impulsePermanentUnban)

        self.GenericMsgHeaderMngr.setdefault("STRING:CHAT", self.impulseChat)
        self.GenericMsgHeaderMngr.setdefault("STRING:TELL", self.impulseTell)
        self.GenericMsgHeaderMngr.setdefault("STRING:FAR_TELL", self.impulseFarTell)
        self.GenericMsgHeaderMngr.setdefault("STRING:CHAT2", self.impulseChat2)
        self.GenericMsgHeaderMngr.setdefault("STRING:DYN_STRING", self.impulseDynString)
        self.GenericMsgHeaderMngr.setdefault("STRING:DYN_STRING_GROUP", self.inpulseDynStringInChatGroup)
        self.GenericMsgHeaderMngr.setdefault("STRING:TELL2", self.impulseTell2)

        self.GenericMsgHeaderMngr.setdefault("TP:DEST", self.impulseTP)
        self.GenericMsgHeaderMngr.setdefault("TP:DEST_WITH_SEASON", self.impulseTPWithSeason)
        self.GenericMsgHeaderMngr.setdefault("TP:CORRECT", self.impulseCorrectPos)
        self.GenericMsgHeaderMngr.setdefault("COMBAT:ENGAGE_FAILED", self.impulseCombatEngageFailed)
        self.GenericMsgHeaderMngr.setdefault("BOTCHAT:DYNCHAT_OPEN", self.impulseDynChatOpen)
        self.GenericMsgHeaderMngr.setdefault("BOTCHAT:DYNCHAT_CLOSE", self.impulseDynChatClose)

        self.GenericMsgHeaderMngr.setdefault("CASTING:BEGIN", self.impulseBeginCast)
        self.GenericMsgHeaderMngr.setdefault("TEAM:INVITATION", self.impulseTeamInvitation)
        self.GenericMsgHeaderMngr.setdefault("TEAM:SHARE_OPEN", self.impulseTeamShareOpen)
        self.GenericMsgHeaderMngr.setdefault("TEAM:SHARE_INVALID", self.impulseTeamShareInvalid)
        self.GenericMsgHeaderMngr.setdefault("TEAM:SHARE_CLOSE", self.impulseTeamShareClose)
        self.GenericMsgHeaderMngr.setdefault("TEAM:CONTACT_INIT", self.impulseTeamContactInit)
        self.GenericMsgHeaderMngr.setdefault("TEAM:CONTACT_CREATE", self.impulseTeamContactCreate)
        self.GenericMsgHeaderMngr.setdefault("TEAM:CONTACT_STATUS", self.impulseTeamContactStatus)
        self.GenericMsgHeaderMngr.setdefault("TEAM:CONTACT_REMOVE", self.impulseTeamContactRemove)

        self.GenericMsgHeaderMngr.setdefault("EXCHANGE:INVITATION", self.impulseExchangeInvitation)
        self.GenericMsgHeaderMngr.setdefault("EXCHANGE:CLOSE_INVITATION", self.impulseExchangeCloseInvitation)
        self.GenericMsgHeaderMngr.setdefault("ANIMALS:MOUNT_ABORT", self.impulseMountAbort)

        self.GenericMsgHeaderMngr.setdefault("DEBUG:REPLY_WHERE", self.impulseWhere)
        self.GenericMsgHeaderMngr.setdefault("DEBUG:COUNTER", self.impulseCounter)


        self.GenericMsgHeaderMngr.setdefault("STRING_MANAGER:PHRASE_SEND", self.impulsePhraseSend)
        self.GenericMsgHeaderMngr.setdefault("STRING_MANAGER:STRING_RESP", self.impulseStringResp)
        self.GenericMsgHeaderMngr.setdefault("STRING_MANAGER:RELOAD_CACHE", self.impulseReloadCache)

        self.GenericMsgHeaderMngr.setdefault("BOTCHAT:FORCE_END", self.impulseBotChatForceEnd)

        self.GenericMsgHeaderMngr.setdefault("JOURNAL:INIT_COMPLETED_MISSIONS", self.impulseJournalInitCompletedMissions)
        self.GenericMsgHeaderMngr.setdefault("JOURNAL:UPDATE_COMPLETED_MISSIONS", self.impulseJournalUpdateCompletedMissions)


        self.GenericMsgHeaderMngr.setdefault("JOURNAL:ADD_COMPASS", self.impulseJournalAddCompass)
        self.GenericMsgHeaderMngr.setdefault("JOURNAL:REMOVE_COMPASS", self.impulseJournalRemoveCompass)

        self.GenericMsgHeaderMngr.setdefault("GUILD:JOIN_PROPOSAL", self.impulseGuildJoinProposal)

        self.GenericMsgHeaderMngr.setdefault("GUILD:ASCENSOR", self.impulseGuildAscensor)
        self.GenericMsgHeaderMngr.setdefault("GUILD:LEAVE_ASCENSOR", self.impulseGuildLeaveAscensor)
        self.GenericMsgHeaderMngr.setdefault("GUILD:ABORT_CREATION", self.impulseGuildAbortCreation)
        self.GenericMsgHeaderMngr.setdefault("GUILD:OPEN_GUILD_WINDOW", self.impulseGuildOpenGuildWindow)

        self.GenericMsgHeaderMngr.setdefault("GUILD:OPEN_INVENTORY", self.impulseGuildOpenInventory)
        self.GenericMsgHeaderMngr.setdefault("GUILD:CLOSE_INVENTORY", self.impulseGuildCloseInventory)

        self.GenericMsgHeaderMngr.setdefault("GUILD:UPDATE_PLAYER_TITLE", self.impulseGuildUpdatePlayerTitle)
        self.GenericMsgHeaderMngr.setdefault("GUILD:USE_FEMALE_TITLES", self.impulseGuildUseFemaleTitles)


        self.GenericMsgHeaderMngr.setdefault("HARVEST:CLOSE_TEMP_INVENTORY", self.impulseCloseTempInv)

        self.GenericMsgHeaderMngr.setdefault("COMMAND:REMOTE_ADMIN", self.impulseRemoteAdmin)

        self.GenericMsgHeaderMngr.setdefault("PHRASE:DOWNLOAD", self.impulsePhraseDownLoad)
        self.GenericMsgHeaderMngr.setdefault("PHRASE:CONFIRM_BUY", self.impulsePhraseConfirmBuy)
        self.GenericMsgHeaderMngr.setdefault("PHRASE:EXEC_CYCLIC_ACK", self.impulsePhraseAckExecuteCyclic)
        self.GenericMsgHeaderMngr.setdefault("PHRASE:EXEC_NEXT_ACK", self.impulsePhraseAckExecuteNext)

        self.GenericMsgHeaderMngr.setdefault("ITEM_INFO:SET", self.impulseItemInfoSet)
        self.GenericMsgHeaderMngr.setdefault("ITEM_INFO:REFRESH_VERSION", self.impulseItemInfoRefreshVersion)
        self.GenericMsgHeaderMngr.setdefault("MISSION_PREREQ:SET", self.impulsePrereqInfoSet)
        self.GenericMsgHeaderMngr.setdefault("ITEM:OPEN_ROOM_INVENTORY", self.impulseItemOpenRoomInventory)
        self.GenericMsgHeaderMngr.setdefault("ITEM:CLOSE_ROOM_INVENTORY", self.impulseItemCloseRoomInventory)

        self.GenericMsgHeaderMngr.setdefault("DEATH:RESPAWN_POINT", self.impulseDeathRespawnPoint)
        self.GenericMsgHeaderMngr.setdefault("DEATH:RESPAWN", self.impulseDeathRespawn)

        self.GenericMsgHeaderMngr.setdefault("DUEL:INVITATION", self.impulseDuelInvitation)
        self.GenericMsgHeaderMngr.setdefault("DUEL:CANCEL_INVITATION", self.impulseDuelCancelInvitation)

        self.GenericMsgHeaderMngr.setdefault("PVP_CHALLENGE:INVITATION", self.impulsePVPChallengeInvitation)
        self.GenericMsgHeaderMngr.setdefault("PVP_CHALLENGE:CANCEL_INVITATION", self.impulsePVPChallengeCancelInvitation)

        self.GenericMsgHeaderMngr.setdefault("PVP_FACTION:PUSH_FACTION_WAR", self.impulsePVPFactionPushFactionWar)
        self.GenericMsgHeaderMngr.setdefault("PVP_FACTION:POP_FACTION_WAR", self.impulsePVPFactionPopFactionWar)
        self.GenericMsgHeaderMngr.setdefault("PVP_FACTION:FACTION_WARS", self.impulsePVPFactionFactionWars)




        self.GenericMsgHeaderMngr.setdefault("ENCYCLOPEDIA:UPDATE", self.impulseEncyclopediaUpdate)
        self.GenericMsgHeaderMngr.setdefault("ENCYCLOPEDIA:INIT", self.impulseEncyclopediaInit)

        self.GenericMsgHeaderMngr.setdefault("USER:BARS", self.impulseUserBars)
        self.GenericMsgHeaderMngr.setdefault("USER:POPUP", self.impulseUserPopup)


        self.GenericMsgHeaderMngr.setdefault("MISSION:ASK_ENTER_CRITICAL", self.impulseEnterCrZoneProposal)
        self.GenericMsgHeaderMngr.setdefault("MISSION:CLOSE_ENTER_CRITICAL", self.impulseCloseEnterCrZoneProposal)


        self.GenericMsgHeaderMngr.setdefault( "MODULE_GATEWAY:FEOPEN", self.cbImpulsionGatewayOpen)
        self.GenericMsgHeaderMngr.setdefault( "MODULE_GATEWAY:GATEWAY_MSG", self.cbImpulsionGatewayMessage )
        self.GenericMsgHeaderMngr.setdefault( "MODULE_GATEWAY:FECLOSE", self.cbImpulsionGatewayClose )

        self.GenericMsgHeaderMngr.setdefault( "OUTPOST:CHOOSE_SIDE", self.impulseOutpostChooseSide )
        self.GenericMsgHeaderMngr.setdefault( "OUTPOST:DECLARE_WAR_ACK", self.impulseOutpostDeclareWarAck )

        self.GenericMsgHeaderMngr.setdefault( "COMBAT:FLYING_ChaScore1_DELTA", self.impulseCombatFlyingChaScore1Delta )
        self.GenericMsgHeaderMngr.setdefault( "COMBAT:FLYING_TEXT_ISE", self.impulseCombatFlyingTextItemSpecialEffectProc )
        self.GenericMsgHeaderMngr.setdefault( "COMBAT:FLYING_TEXT", self.impulseCombatFlyingText )

        self.GenericMsgHeaderMngr.setdefault( "SEASON:SET", self.impulseSetSeason )
        self.GenericMsgHeaderMngr.setdefault( "RING_MISSION:DSS_DOWN", self.impulseDssDown )

        self.GenericMsgHeaderMngr.setdefault( "NPC_ICON:SET_DESC", self.impulseSetNpcIconDesc )
        self.GenericMsgHeaderMngr.setdefault( "NPC_ICON:SVR_EVENT_MIS_AVL", self.impulseServerEventForMissionAvailability )
        self.GenericMsgHeaderMngr.setdefault( "NPC_ICON:SET_TIMER", self.impulseSetNpcIconTimer )

    def sizeElement(self, keys = None):
        head = self.msgXml
        if not keys:
            return len(head)
        ret = len(head)
        for key in keys.split(':'):
            for ele in head:
                if ele.attrib['name'] == key:
                    head = ele
                    ret = len(head)
                    break
        return ret

    def searchElement(self, keys):
        ret = []
        head = self.msgXml
        self.log.debug(len(head))
        for key in keys.split(':'):
            #print(key)
            id = 0
            for ele in head:
                if ele.attrib['name'] == key:
                    self.log.debug('%s => %d' % (key, id))
                    ret.append(id)
                    head = ele
                    break
                id += 1
        return ret

    def execute(self, msgin):
        head = self.msgXml
        listpath = []
        while True:
            nbBit = getPowerOf2(len(head))
            #  def readSerial(self, nbits, name="", decode=True, typeName='', emulate=False):
            id = msgin.readSerial(nbBit, name='Ptr', typeName='Number', emulate=True)

            ele = head[id]
            name = ele.attrib['name']
            listpath.append(name)
            fullname = ':'.join(listpath)

            id = msgin.readSerial(nbBit, name='Ptr', typeName='MsgXML<' + name + '>')
            if fullname in self.GenericMsgHeaderMngr:
                self.log.debug("Found : %s" % fullname)
                self.GenericMsgHeaderMngr[fullname](msgin)
                self.log.debug("MessageXML decoded: %s" % msgin.showAllData() )
                return True
            else:
                #self.log.debug("Non trouve")
                for ele in head:
                    if ele.attrib['name'] == name:
                        head = ele
                        break
                if head != ele:
                    self.log.error("Impossible to found %s" % fullname )
                    self.log.debug("MessageXML decoded: %s" % msgin.showAllData() )
                    return False
        self.log.debug("MessageXML decoded: %s" % msgin.showAllData() )
        return False

    def loadMsg(self, msgXml):
        self.msgXml = msgXml

    def loadDatabase(self, databaseXml):
        self.databaseXml = databaseXml

class CAction:
    def __init__(self, slot, code, world):
        self.Code = code
        self.PropertyCode = code
        self.Slot = slot
        self._Priority = 1
        self.Timeout = 0
        self.GameCycle = 0
        self.world = world

    def unpack(self, message):
        raise RuntimeError

    def pack(self, message):
        raise RuntimeError

    def serialIn(self, msgin):
        raise RuntimeError

    def serialOut(self, msgout):
        raise RuntimeError

    def size(self):
        raise RuntimeError

    def getMaxSizeInBit(self):
        raise RuntimeError

    def setPriority(self, prio):
        raise RuntimeError

    def priority(self):
        raise RuntimeError

    def getValue(self):
        raise RuntimeError

    def setValue(self, value):
        raise RuntimeError

    def isContinuous(self):
        raise RuntimeError

    def reset(self):
        raise RuntimeError

    def __str__(self):
        return "[%d,%d]" % (self.Code , self.Slot)

class CActionPosition(CAction):
    def __init__(self, slot, code, world):
        super().__init__(slot, code, world)

    def __str__(self):
        return "CActionPosition" + super().__str__()

class CActionSync(CAction):
    def __init__(self, slot, code, world):
        super().__init__(slot, code, world)

    def __str__(self):
        return "CActionSync" + super().__str__()

class CActionDisconnection(CAction):
    def __init__(self, slot, code, world):
        super().__init__(slot, code, world)

    def __str__(self):
        return "CActionDisconnection" + super().__str__()

class CActionAssociation(CAction):
    def __init__(self, slot, code, world):
        super().__init__(slot, code, world)

    def __str__(self):
        return "CActionAssociation" + super().__str__()

class CActionDummy(CAction):
    def __init__(self, slot, code, world):
        super().__init__(slot, code, world)

    def __str__(self):
        return "CActionDummy" + super().__str__()

class CActionLogin(CAction):
    def __init__(self, slot, code, world):
        super().__init__(slot, code, world)

    def __str__(self):
        return "CActionLogin" + super().__str__()

class CActionTargetSlot(CAction):
    def __init__(self, slot, code, world):
        super().__init__(slot, code, world)

    def __str__(self):
        return "CActionTargetSlot" + super().__str__()

class CActionGeneric(CAction):
    def __init__(self, slot, code, world):
        super().__init__(slot, code, world)
        self._Message = None

    def unpack(self, message):
        size = message.readUint32('size')
        self._Message = message.readBitStreamUint8(size, 'message')

    def pack(self, msgout):
        msgout.pushArrayUint8(self._Message, len(self._Message) )

    def reset(self):
        self._Message = None

    def genericAction(self, decodeImpulse):
        decodeImpulse.execute(self._Message)

    def __str__(self):
        return "CActionGeneric" + super().__str__() + "[" + self._Message.showAllData() + ']'

class CActionGenericMultiPart(CAction):
    '''
        khanat-opennel-code/code/ryzom/common/src/game_share/action_generic_multi_part.h # class CActionGenericMultiPart
    '''
    def __init__(self, slot, code, world):
        super().__init__(slot, code, world)
        self.PartCont = []
        self.Number = 0
        self.Part = 0
        self.NbBlock = 0

    def unpack(self, message):
        self.Number = message.readUint8('Number')
        self.Part = message.readUint16('Part')
        self.NbBlock = message.readUint16('NbBlock')

        size = message.readUint32('size')
        self.PartCont = message.readBitStreamUint8(size, 'PartCont')


    def pack(self, msgout):
        msgout.pushUint8(self.Number)
        msgout.pushUint16(self.Part)
        msgout.pushUint16(self.NbBlock)
        msgout.pushArrayUint8(self.PartCont)

    def reset(self):
        self.PartCont = []
        self.Number = 0
        self.Part = 0
        self.NbBlock = 0

    def genericAction(self, decodeImpulse):
        '''
            khanat-opennel-code/code/ryzom/client/src/network_connection.cpp # void CNetworkConnection::genericAction (CActionGenericMultiPart *agmp)
        '''
        self.log = logging.getLogger('myLogger')
        self.log.debug("Number:%d Part:%d NbBlock:%d" % (self.Number, self.Part, self.NbBlock))
        self.world.addGenericMultiPartTemp(self.Number)
        self.world.setGenericMultiPartTemp(self.Number, self.Part, self.NbBlock, self.PartCont)

    def __str__(self):
        return "CActionGenericMultiPart" + super().__str__() + "[" + str(self.Number) + ',' + str(self.Part) + ',' + str(self.NbBlock) + ',' + self.PartCont.showAllData() + ']'

class CActionSint64(CAction):
    def __init__(self, slot, code, world):
        super().__init__(slot, code, world)

    def __str__(self):
        return "CActionSint64" + super().__str__()

class CActionFactory:
    def __init__(self, log, world):
        self.log = log
        self.world = world
        self.RegisteredAction = {}
        self.RegisteredAction.setdefault(TActionCode.ACTION_POSITION_CODE, [])
        self.RegisteredAction.setdefault(TActionCode.ACTION_GENERIC_CODE, [])
        self.RegisteredAction.setdefault(TActionCode.ACTION_GENERIC_MULTI_PART_CODE, [])
        self.RegisteredAction.setdefault(TActionCode.ACTION_SINT64, [])
        self.RegisteredAction.setdefault(TActionCode.ACTION_SYNC_CODE, [])
        self.RegisteredAction.setdefault(TActionCode.ACTION_DISCONNECTION_CODE, [])
        self.RegisteredAction.setdefault(TActionCode.ACTION_ASSOCIATION_CODE, [])
        self.RegisteredAction.setdefault(TActionCode.ACTION_LOGIN_CODE, [])
        self.RegisteredAction.setdefault(TActionCode.ACTION_TARGET_SLOT_CODE, [])
        self.RegisteredAction.setdefault(TActionCode.ACTION_DUMMY_CODE, [])

    def createFactory(self, slot, code):
        if code == TActionCode.ACTION_POSITION_CODE:
            self.log.debug("Create CActionPosition")
            return CActionPosition(slot, code, self.world)
        elif code == TActionCode.ACTION_GENERIC_CODE:
            self.log.debug("Create CActionGeneric")
            return CActionGeneric(slot, code, self.world)
        elif code == TActionCode.ACTION_GENERIC_MULTI_PART_CODE:
            self.log.debug("Create CActionGenericMultiPart")
            return CActionGenericMultiPart(slot, code, self.world)
        elif code == TActionCode.ACTION_SINT64:
            self.log.debug("Create CActionSint64")
            return CActionSint64(slot, code, self.world)
        elif code == TActionCode.ACTION_SYNC_CODE:
            self.log.debug("Create CActionSync")
            return CActionSync(slot, code, self.world)
        elif code == TActionCode.ACTION_DISCONNECTION_CODE:
            self.log.debug("Create CActionDisconnection")
            return CActionDisconnection(slot, code, self.world)
        elif code == TActionCode.ACTION_ASSOCIATION_CODE:
            self.log.debug("Create CActionAssociation")
            return CActionAssociation(slot, code, self.world)
        elif code == TActionCode.ACTION_LOGIN_CODE:
            self.log.debug("Create CActionLogin")
            return CActionLogin(slot, code, self.world)
        elif code == TActionCode.ACTION_TARGET_SLOT_CODE:
            self.log.debug("Create CActionTargetSlot")
            return CActionTargetSlot(slot, code, self.world)
        elif code == TActionCode.ACTION_DUMMY_CODE:
            self.log.debug("Create CActionDummy")
            return CActionDummy(slot, code, self.world)
        else:
            log = logging.getLogger('myLogger')
            log.warning('create() try to create an unknown action (%u)' % code)
            raise RuntimeError

    def create(self, slot, code):
        if code not in self.RegisteredAction:
            log = logging.getLogger('myLogger')
            log.warning('try to create an unknown action (%u)' % code)
            raise None
        elif not self.RegisteredAction[code]:
            self.log.debug("new CAction")
            action = self.createFactory(slot, code)
            action.reset()
            return action
        else:
            self.log.debug("update CAction")
            action = self.RegisteredAction[code][-1]
            action.reset()
            action.PropertyCode = code
            action.Slot = slot
            return action

    def unpack(self, msgin):
        '''
            khanat-opennel-code/code/ryzom/common/src/game_share/action_factory.cpp : CAction *CActionFactory::unpack (NLMISC::CBitMemStream &message, NLMISC::TGameCycle /* currentCycle */ )
        '''
        if msgin.needRead() >= 8:
            shortcode = msgin.readBool('shortcode')
            if shortcode:
                code = msgin.readSerial(2, 'code')
            else:
                code = msgin.readUint8('code')
            action = self.create(INVALID_SLOT, code)
            if action:
                try:
                    action.unpack (msgin);
                except RuntimeError:
                    log = logging.getLogger('myLogger')
                    log.warning('Missing code to unpack (code :%u)' % code)
                    raise RuntimeError
            else:
                log = logging.getLogger('myLogger')
                log.warning('Unpacking an action with unknown code, skip it (%u)' % code)
        return action
    def pack(self, action, msgout):
        if action.Code < 4:
            msgout.pushBool(True)
            msgout.internalSerial(self.Code, 2)
        else:
            msgout.pushBool(False)
            msgout.pushUint8(self.Code)
        action.pack(message)

class CActionBlock:
    def __init__(self):
        self.Cycle = 0
        self.FirstPacket = 0
        self.Actions = None
        self.Sucess = True

    def serial(self, msgout, actionFactory):
        msgout.pushUint32(self.Cycle)
        msgout.pushUint8(len(self.Actions))
        for action in self.Actions:
            # msgPosBefore = msgout.getPosInBit()
            actionFactory.pack(action, msgout)
            # msgPosAfter = msgout.getPosInBit()
            # actionSize = actionFactory.size(action)

class CImpulseDecoder:
    '''
        see : khanat-opennel-code/code/ryzom/client/src/impulse_decoder.cpp
    '''
    def __init__(self, log, world):
        self.log = log
        self.world = world
        self.reset()
        self._CActionFactory = CActionFactory(log, world)

    def removeCAction(self, action):
        self._CActionFactory.RegisteredAction[action.Code].append(action)

    def decode(self, msgin, receivedPacket, receivedAck, nextSentPacket):
        self.log.debug("receivedPacket:%d receivedAck:%d nextSentPacket:%d" %(receivedPacket, receivedAck, nextSentPacket))
        actions = []
        for level in range(0, 3):
            if level == 0:
                lAck = self._LastAck0
                channel = 0
            elif level == 1:
                lAck = self._LastAck1
                channel = receivedPacket & 1
            elif level == 2:
                lAck = self._LastAck2
                channel = receivedPacket & 3
            keep = True
            checkOnce = False
            num = 0
            self.log.debug("channel:%d lAck:%s" %(channel, ':'.join([str(x) for x in lAck])))
            # lastAck = lAck[channel]
            while True:
                next = msgin.readBool('next:' + str(level) + ':' + str(channel))
                if not next:
                    break
                if not checkOnce:
                    checkOnce = True
                    keep = receivedAck >= lAck[channel]
                    self.log.debug("keep:%s" % str(keep))
                    if keep:
                        lAck[channel] = nextSentPacket
                        self.log.debug("lAck:%s" % ':'.join([str(x) for x in lAck]))
                num += 1
                action = self._CActionFactory.unpack(msgin)
                if keep:
                    self.log.debug("keep")
                    actions.append(action)
                elif action:
                    self.log.debug("append")
                    self.removeCAction(action)
        return actions

    def reset(self):
        self._LastAck0 = [-1]
        self._LastAck1 = [-1, -1]
        self._LastAck2 = [-1, -1, -1, -1]

class TStampQueue:
    def __init__(self, first=None, second=None):
        self.first = first
        self.second = second

class ClientNetworkConnection:
    '''
    Partie client de la gestion de la communication reseau avec le serveur:

    client :
        code/ryzom/client/src/network_connection.cpp
    server :
        khanat-opennel-code/code/ryzom/server/src/frontend_service/fe_receive_sub.cpp # void CFeReceiveSub::handleReceivedMsg( CClientHost *clienthost )
    '''
    def __init__(self,
                 khanat_host,
                 khanat_port_frontend,
                 LanguageCode="fr",
                 checkMessageNumber = True):
        self.log = logging.getLogger('myLogger')
        self._CurrentSendNumber = 0
        self.LanguageCode = LanguageCode
        self._QuitId = 0
        self._ConnectionState = TConnectionState.NotInitialised
        self.UserAddr, self.UserKey, self.UserId = None, None, None
        self.frontend = (khanat_host, khanat_port_frontend)
        self._sock = None
        self._CurrentReceivedNumber = 0
        self._SystemMode = 0
        self._LastReceivedAck = 0
        self._LastReceivedNumber = 0
        self._LastAckInLongAck = 0
        self._MsgXmlMD5 = None
        self._DatabaseXmlMD5 = None
        self.msgXml = None
        self.databaseXml = None
        self._Synchronize = 0
        self._LatestSync = 0
        self._CurrentServerTick = 0
        self._MsPerTick = 0
        self._LCT = 100
        self._UpdateTime = 0
        #self._UpdateTicks = 0
        self._ReceivedSync = False
        self._LastReceivedTime = 0
        self._LastReceivedPacketInBothModes = 0
        self._TotalMessages = 0
        self._TotalLostPackets = 0
        self.checkMessageNumber = checkMessageNumber
        self._LastAckBit = 0
        self._AckBitMask = 0
        self._LongAckBitField = CBitSet()
        self._LatestSyncTime = 0
        self.world = World(self.log)
        self._ImpulseDecoder = CImpulseDecoder(self.log, self.world)
        self._LongAckBitField.resize(1024)
        self._LatestProbeTime = 0
        self._LatestProbe = 0
        self._LatestProbes = []
        self._LatestQuitTime = 0
        self._ReceivedAckQuit = False
        self._Actions = []
        self._PacketStamps = []
        self.decodeImpulse = DecodeImpulse(self.log, self.world)
        self._InstantPing = 0
        self._BestPing = 10000
        self._MsPerTick = 100
        self._LastSendTime = 0

    def signal_exit(self, sig, frame):
        self.log.warning("Receive signal to quit program")
        self.sendSystemQuit()
        sys.exit(0)

    def connect(self):
        signal.signal(signal.SIGINT, self.signal_exit)
        signal.signal(signal.SIGTERM, self.signal_exit)
        try:
            self._sock = socket.socket(socket.AF_INET, # Internet
                                socket.SOCK_DGRAM) # UDP
        except:
            self.log.error("Impossible to connect on khanat")
            return False
        self._ConnectionState = TConnectionState.Login
        self._LatestSyncTime = int(time.clock_gettime(1)*1000)
        return True

    def cookiesInit(self, UserAddr, UserKey, UserId):
        self.UserAddr = UserAddr
        self.UserKey = UserKey
        self.UserId = UserId

    def reset(self):
        self._CurrentSendNumber += 0

    def buildSystemHeader(self, msgout): # code/ryzom/client/src/network_connection.cpp # void CNetworkConnection::buildSystemHeader(NLMISC::CBitMemStream &msgout)
        msgout.pushSint32(self._CurrentSendNumber)
        msgout.pushBool(True) # systemMode
        self._PacketStamps.append( TStampQueue(self._CurrentSendNumber, self._UpdateTime) )
        self._CurrentSendNumber += 1

    def sendSystemLogin(self): # code/ryzom/client/src/network_connection.cpp # void    CNetworkConnection::sendSystemLogin()
        if self._sock is None:
            raise ValueError
        msgout = BitStream()
        self.buildSystemHeader(msgout)
        msgout.pushUint8(CLFECOMMON.SYSTEM_LOGIN_CODE)
        msgout.pushUint32(self.UserAddr)
        msgout.pushUint32(self.UserKey)
        msgout.pushUint32(self.UserId)
        msgout.pushString(self.LanguageCode)

        self._sock.sendto(msgout.toBytes(), self.frontend)
        self._CurrentSendNumber += 1

        self._ConnectionState = TConnectionState.Login

    def sendSystemQuit(self): # code/ryzom/client/src/network_connection.cpp # void CNetworkConnection::sendSystemQuit()
        # Disconnect
        if self._sock is None:
            raise ValueError
        self._QuitId += 1
        msgout = BitStream()
        self.buildSystemHeader(msgout)
        msgout.pushUint8(CLFECOMMON.SYSTEM_QUIT_CODE)
        msgout.pushSint32(self._QuitId) # _QuitId
        self._sock.sendto(msgout.toBytes(), self.frontend)
        self._ConnectionState = TConnectionState.Quit

    def sendSystemAckSync(self): # code/ryzom/client/src/network_connection.cpp # void  CNetworkConnection::sendSystemAckSync()
        self.log.debug("sendSystemAckSync")
        msgout = BitStream()
        self.buildSystemHeader(msgout)
        msgout.pushUint8(CLFECOMMON.SYSTEM_ACK_SYNC_CODE)
        msgout.pushSint32(self._LastReceivedNumber)
        msgout.pushSint32(self._LastAckInLongAck)
        self._LongAckBitField.writeSerial(msgout) # Signale le nombre de packet perdu
        msgout.pushSint32(self._LatestSync)
        self._sock.sendto(msgout.toBytes(), self.frontend)
        self._LatestSyncTime = self._UpdateTime

    def sendSystemAckProbe(self):
        self.log.debug("sendSystemAckProbe")
        msgout = BitStream()
        self.buildSystemHeader(msgout)
        msgout.pushUint8(CLFECOMMON.SYSTEM_ACK_PROBE_CODE)
        msgout.pushSint32(len(self._LatestProbes))
        for data in self._LatestProbes:
            msgout.pushSint32(data)
        self._LatestProbes = []
        self._sock.sendto(msgout.toBytes(), self.frontend)


    def sendSystemDisconnection(self):
        self.log.debug("sendSystemDisconnection")
        if self._sock is None:
            raise ValueError
        msgout = BitStream()
        self.buildSystemHeader(msgout)
        msgout.pushUint8(CLFECOMMON.SYSTEM_DISCONNECTION_CODE)
        self._sock.sendto(msgout.toBytes(), self.frontend)

    def sendNormalMessage(self):
        self.log.debug("sendNormalMessage")
        if self._sock is None:
            raise ValueError
        msgout = BitStream()
        msgout.pushSint32(self._CurrentSendNumber)
        msgout.pushBool(False) # Normal
        msgout.pushSint32(self._LastReceivedNumber)
        msgout.pushSint32(self._AckBitMask)

        numPacked = 0

        for block in self._Actions:
            if block.Cycle == 0:
                break

            if block.FirstPacket == 0:
                block.FirstPacket = self._CurrentSendNumber;

            block.serial(message)
            numPacked += 1
            if message.getPosInBit() > 480*8: # easy version
                break

        self._sock.sendto(msgout.toBytes(), self.frontend)

        self._LastSendTime = int(time.clock_gettime(1)*1000)
        self._PacketStamps.append( TStampQueue(self._CurrentSendNumber, self._UpdateTime) )
        self._CurrentSendNumber += 1

    def readDelta(self, msg):
        propertyCount = msg.readUint16('propertyCount')
        self.log.debug("propertyCount:%d" % propertyCount)
        self.log.debug("TODO")
        for _ in range(0, propertyCount):
            pass

    def buildStream(self, buffersize=65536):
        # khanat-opennel-code/code/ryzom/client/src/network_connection.cpp # bool CNetworkConnection::buildStream( CBitMemStream &msgin )
        data, addr = self._sock.recvfrom(buffersize)
        return data, addr

    def decodeHeader(self, msg):
        self._TotalMessages += 1
        self._LastReceivedTime = self._UpdateTime
        self._CurrentReceivedNumber = msg.readSint32('CurrentReceivedNumber')
        self._SystemMode = msg.readBool('SystemMode')

        if self.checkMessageNumber and self._CurrentReceivedNumber > self._LastReceivedPacketInBothModes:
            self._TotalLostPackets += self._CurrentReceivedNumber - self._LastReceivedPacketInBothModes - 1
            self._LastReceivedPacketInBothModes = self._CurrentReceivedNumber
#        else:
#            self._LastReceivedPacketInBothModes = self._CurrentReceivedNumber - 1

        if not self._SystemMode:
            self._LastReceivedAck = msg.readSint32('LastReceivedAck');
            self.log.debug("Normal Mode _LastReceivedAck:%d" % self._LastReceivedAck)
        else:
            self.log.debug("System Mode")

        if self._CurrentReceivedNumber > self._LastReceivedNumber+1:
            self.log.debug("lost messages server->client [%d; %d]" %(self._LastReceivedPacketInBothModes + 1, self._CurrentReceivedNumber - 1))
        elif self._CurrentReceivedNumber == self._LastReceivedNumber:
            self.log.debug("awaiting packet %d, received packet %d" %(self._LastReceivedPacketInBothModes + 1, self._CurrentReceivedNumber))
            return False
        elif self._CurrentReceivedNumber < self._LastReceivedNumber:
            self.log.debug("received an old message, awaiting packet %d, received packet %d" %(self._LastReceivedPacketInBothModes + 1, self._CurrentReceivedNumber))
            return false
        ackBool = ( not self._SystemMode ) and (self._ConnectionState == TConnectionState.Connected or self._ConnectionState == TConnectionState.Synchronize)
        if ackBool:
            ackBit = 1
        else:
            ackBit = 0
        if self._CurrentReceivedNumber - self._LastReceivedNumber < 32:
            self._AckBitMask <<= self._CurrentReceivedNumber - self._LastReceivedNumber;
        elif (self_CurrentReceivedNumber - self_LastReceivedNumber) == 32 and self._LastAckBit != 0:
            self._AckBitMask = 0x80000000
        else:
            self._AckBitMask = 0x00000000
        self._LastAckBit = ackBit;
        for i in range(self._LastReceivedNumber+1, self._CurrentReceivedNumber+1):
            self._LongAckBitField.clearBit(i & 511)  # (512 - 1) mask 9bit
        self._LongAckBitField.set(self._CurrentReceivedNumber & 511, ackBool) # (512 - 1) mask 9bit

        if self._LastAckInLongAck <= (self._CurrentReceivedNumber-512):
            self._LastAckInLongAck = self._CurrentReceivedNumber-511; # (512 - 1) mask 9bit

        self._LastReceivedNumber = self._CurrentReceivedNumber
        self.log.debug("_CurrentReceivedNumber:%d, _LastReceivedNumber:%d, ackBit:%d, _AckBitMask:%d _LongAckBitField:%s" % (self._CurrentReceivedNumber, self._LastReceivedNumber, ackBit, self._AckBitMask, self._LongAckBitField))
        return True

    def receiveSystemProbe(self, msg):
        self._LatestProbeTime = self._UpdateTime
        self._LatestProbe = msg.readSint32('LatestProbe')
        self.log.debug("LatestProbe: %d" % self._LatestProbe)
        self._LatestProbes.append(self._LatestProbe)

    def receiveSystemStalled(self, msg):
        self.log.debug("received STALLED")

    def receiveSystemSync(self, msg):
        self._LatestSyncTime = self._UpdateTime
        self._Synchronize = msg.readUint32('Synchronize')
        stime = msg.readSint64('stime')
        self._LatestSync = msg.readUint32('LatestSync')
        self.log.debug("%d %d %d" %(self._Synchronize, stime, self._LatestSync))
        # khanat-opennel-code/code/ryzom/client/src/network_connection.cpp : void CNetworkConnection::receiveSystemSync(CBitMemStream &msgin)
        MsgData = msg.readArrayUint8(16, 'MsgData')
        DatabaseData = msg.readArrayUint8(16, 'DatabaseData')
        self.log.debug("MsgData:" + str(MsgData))
        self.log.debug("DatabaseData:" + str(DatabaseData))
        md5Msg = bytes(MsgData)
        md5Database = bytes(DatabaseData)
        if md5Msg == self._MsgXmlMD5:
            self.log.info("Check MD5 msg.xml : OK")
        else:
            self.log.error("Check MD5 msg.xml : KO")
        if md5Database == self._DatabaseXmlMD5:
            self.log.info("Check MD5 database.xml : OK")
        else:
            self.log.error("Check MD5 database.xml : KO")
        self._MsPerTick = 100
        self._CurrentServerTick = self._Synchronize + self._CurrentReceivedNumber + 2
        self._CurrentClientTick = self._CurrentServerTick - ( self._LCT + self._MsPerTick ) / self._MsPerTick
        self._CurrentClientTime = self._UpdateTime - (self._LCT + self._MsPerTick)
        self.sendSystemAckSync()

    def decodeVisualProperties(self, msgin):
        '''
            khanat-opennel-code/code/ryzom/client/src/network_connection.cpp # void	CNetworkConnection::decodeVisualProperties( CBitMemStream& msgin )
        '''
        while True:
            if msgin.getPosInBit() + 8*8 > len(msgin)*8:
                return
            slot = msgin.readUint8('slot')
            associationBits = msgin.readUint32('associationBits')
            self.log.debug("TODO")
            return

    def receiveNormalMessage(self, msgin):
        self.log.debug("received normal message Packet (%d) %s" % (msgin.needRead(), msgin.showLastData() ))
        actions = self._ImpulseDecoder.decode(msgin, self._CurrentReceivedNumber, self._LastReceivedAck, self._CurrentSendNumber )
        if actions:
            self.log.debug('actions: ' +','.join( [ str(x) for x in actions] ) )
        else:
            self.log.debug('actions: None')
        self.log.debug("Message not read (%d) %s" % (msgin.needRead(), msgin.showLastData() ))

        # remove all old actions that are acked
        while self._Actions and self._Actions[0].FirstPacket != 0 and self._Actions[0].FirstPacket :
            self.log.debug("remove old action")
            self._Actions.pop(0)

        self._CurrentServerTick = self._CurrentReceivedNumber * 2 + self._Synchronize

        # remove useless stamps in queue
        while len(self._PacketStamps) != 0 and self._LastReceivedAck > self._PacketStamps[0].first:
            self._PacketStamps.pop(0)

        # Statistique !
        if len(self._PacketStamps) == 0 or self._PacketStamps[0].first > self._LastReceivedAck:
            pass
        else:
            ackedPacketTime = self._PacketStamps[0].second
            ping = self._UpdateTime - ackedPacketTime
            self._InstantPing = ping
            if ping < self._BestPing:
                self._BestPing = ping
            earliest = ackedPacketTime + self._BestPing//2
            latest = self._UpdateTime - self._BestPing//2

            numStepTick = self._CurrentServerTick - self._CurrentClientTick
            if numStepTick > 0 and earliest > self._CurrentClientTime and latest > self._CurrentClientTime:
                if self._CurrentClientTime + self._MsPerTick * numStepTick < earliest:
                    self._MsPerTick = (earliest - self._CurrentClientTime)//numStepTick
                if (self._CurrentClientTime + self._MsPerTick * numStepTick) > latest:
                    self._MsPerTick = (latest - self._CurrentClientTime)//numStepTick
                if self._MsPerTick == 0:
                    self.log.warning("_MsPerTick is 0 because server tick is too big %d compare to the client tick is %d" %(self._CurrentServerTick, self._CurrentClientTick))
                    self._MsPerTick = 1
            elif numStepTick <= 0:
                self._MsPerTick = self._LCT


        ## remove useless stamps in queue
        #while self._PacketStamps and self._LastReceivedAck > self._PacketStamps[0].first:
        #    self._PacketStamps.pop(0)

        # Decode the actions received in the impulsions
        for action in actions:
            if action.Code == TActionCode.ACTION_DISCONNECTION_CODE:
                self.log.debug("Action : ACTION_DISCONNECTION_CODE")
                self.disconnect()
            elif action.Code == TActionCode.ACTION_GENERIC_CODE:
                self.log.debug("Action : ACTION_GENERIC_CODE")
                action.genericAction(self.decodeImpulse)
            elif action.Code == TActionCode.ACTION_GENERIC_MULTI_PART_CODE:
                self.log.debug("Action : ACTION_GENERIC_MULTI_PART_CODE")
                action.genericAction(self.decodeImpulse)
            elif action.Code == TActionCode.ACTION_DUMMY_CODE:
                self.log.debug("Action : ACTION_DUMMY_CODE")
            self._ImpulseDecoder.removeCAction(action)

        # Decode the visual properties
        self.decodeVisualProperties( msgin );

        self._LastReceivedNormalTime = int(time.clock_gettime(1)*1000)

    def receiveSystemAckQuit(self, msgin):
        self.log.debug("received ACK_QUIT")
        self._ReceivedAckQuit = True


    def disconnect(self):
        self.log.info("Disconnect")
        self.sendSystemDisconnection()
        self._sock.close()
        selc._sock = None
        self._ConnectionState = TConnectionState.Disconnect

    def stateLogin(self, msgin):
        self.decodeHeader(msgin)
        if self._SystemMode:
            message = msgin.readUint8('message')
            self.log.debug("_CurrentReceivedNumber:%d (mode:%s) %d [%d/%d/%d]" % (self._CurrentReceivedNumber, str(self._SystemMode), message, msgin.sizeData(),  msgin.sizeRead(), msgin.needRead()))
            if message == CLFECOMMON.SYSTEM_SYNC_CODE:
                self._ConnectionState = TConnectionState.Synchronize
                self.log.debug("Login->synchronize")
                self.receiveSystemSync(msgin)
                return True
            elif message == CLFECOMMON.SYSTEM_STALLED_CODE:
                self.log.debug("received STALLED")
                self._ConnectionState = TConnectionState.Stalled
                self.receiveSystemStalled(msgin)
                return True
            elif message == CLFECOMMON.SYSTEM_PROBE_CODE:
                self.log.debug("Login->probe")
                self._ConnectionState = TConnectionState.Probe
                self.receiveSystemProbe(msgin)
                return True
            elif message == CLFECOMMON.SYSTEM_SERVER_DOWN_CODE:
                self.disconnect()
                self.log.warning("BACK-END DOWN")
                return False
            else:
                self.log.warning("CNET: received system %d in state Login" % message)
            self.log.debug("_CurrentReceivedNumber:%d (mode:%s) %d [%d/%d/%d] '%s'" % (self._CurrentReceivedNumber, str(self._SystemMode), message, msgin.sizeData(),  msgin.sizeRead(), msgin.needRead(), msgin.showLastData()))
        else:
            self.log.warning("CNET: received normal in state Login")
        return False

    def stateSynchronize(self, msgin):
        self.decodeHeader(msgin)
        if self._SystemMode:
            message = msgin.readUint8('message')
            self.log.debug("_CurrentReceivedNumber:%d (mode:%s) %d [%d/%d/%d]" % (self._CurrentReceivedNumber, str(self._SystemMode), message, msgin.sizeData(),  msgin.sizeRead(), msgin.needRead()))
            if message == CLFECOMMON.SYSTEM_PROBE_CODE:
                self.log.debug("synchronize->probe")
                self._ConnectionState = TConnectionState.Probe
                self.receiveSystemProbe(msgin)
                return True
            elif message == CLFECOMMON.SYSTEM_STALLED_CODE:
                self.log.debug("received STALLED")
                self._ConnectionState = TConnectionState.Stalled
                self.receiveSystemStalled(msgin)
                return True
            elif message == CLFECOMMON.SYSTEM_SYNC_CODE:
                self.log.debug("synchronize->synchronize")
                self.receiveSystemSync(msgin)
            elif message == CLFECOMMON.SYSTEM_SERVER_DOWN_CODE:
                self.disconnect()
                self.log.warning("BACK-END DOWN")
                return False
            else:
                self.log.warning("CNET: received system %d in state Synchronize" % message)
            self.log.debug("_CurrentReceivedNumber:%d (mode:%s) %d [%d/%d/%d] '%s'" % (self._CurrentReceivedNumber, str(self._SystemMode), message, msgin.sizeData(),  msgin.sizeRead(), msgin.needRead(), msgin.showLastData()))
        else:
            self._ConnectionState = TConnectionState.Connected
            self.log.warning("CNET: synchronize->connected")
            # _Changes.push_back(CChange(0, ConnectionReady));
            self.receiveNormalMessage(msgin);
            return True
        if  self._UpdateTime - self._LatestSyncTime > 300:
            self.sendSystemAckSync();
        return False

    def stateConnected(self, msgin):
        self.decodeHeader(msgin)
        if self._SystemMode:
            message = msgin.readUint8('message')
            self.log.debug("_CurrentReceivedNumber:%d (mode:%s) %d [%d/%d/%d]" % (self._CurrentReceivedNumber, str(self._SystemMode), message, msgin.sizeData(),  msgin.sizeRead(), msgin.needRead()))
            if message == CLFECOMMON.SYSTEM_PROBE_CODE:
                self.log.debug("Connected->probe")
                self._ConnectionState = TConnectionState.Probe
                self.receiveSystemProbe(msgin)
                return True
            elif message == CLFECOMMON.SYSTEM_SYNC_CODE:
                self.log.debug("Connected->synchronize")
                self.receiveSystemSync(msgin)
                return True
            elif message == CLFECOMMON.SYSTEM_STALLED_CODE:
                self.log.debug("received STALLED")
                self._ConnectionState = TConnectionState.Stalled
                self.receiveSystemStalled(msgin)
                return True
            elif message == CLFECOMMON.SYSTEM_SERVER_DOWN_CODE:
                self.disconnect()
                self.log.warning("BACK-END DOWN")
                return False
            else:
                self.log.warning("CNET: received system %d in state Connected" % message)
            self.log.debug("_CurrentReceivedNumber:%d (mode:%s) %d [%d/%d/%d] '%s'" % (self._CurrentReceivedNumber, str(self._SystemMode), message, msgin.sizeData(),  msgin.sizeRead(), msgin.needRead(), msgin.showLastData()))
        else:
            self.receiveNormalMessage(msgin);
            return True
        return False

    def stateProbe(self, msgin):
        self.decodeHeader(msgin)
        if self._SystemMode:
            message = msgin.readUint8('message')
            self.log.debug("_CurrentReceivedNumber:%d (mode:%s) %d [%d/%d/%d]" % (self._CurrentReceivedNumber, str(self._SystemMode), message, msgin.sizeData(),  msgin.sizeRead(), msgin.needRead()))
            if message == CLFECOMMON.SYSTEM_SYNC_CODE:
                self.log.debug("probe->synchronize")
                self._ConnectionState = TConnectionState.Synchronize
                self.receiveSystemSync(msgin)
                return True
            elif message == CLFECOMMON.SYSTEM_STALLED_CODE:
                self.log.debug("probe->stalled")
                self._ConnectionState = TConnectionState.Stalled
                self.receiveSystemStalled(msgin)
                return True
            elif message == CLFECOMMON.SYSTEM_PROBE_CODE:
                self.receiveSystemProbe(msgin)
            elif message == CLFECOMMON.SYSTEM_SERVER_DOWN_CODE:
                self.disconnect()
                self.log.warning("BACK-END DOWN")
                return False
            else:
                self.log.warning("CNET: received system %d in state Probe" % message)
            self.log.debug("_CurrentReceivedNumber:%d (mode:%s) %d [%d/%d/%d] '%s'" % (self._CurrentReceivedNumber, str(self._SystemMode), message, msgin.sizeData(),  msgin.sizeRead(), msgin.needRead(), msgin.showLastData()))
        else:
            self.log.warning("received normal in state Probe")

        if (len(self._LatestProbes) > 0) or (self._UpdateTime - self._LatestProbeTime > 300):
            self.sendSystemAckProbe()
            self._LatestProbeTime = self._UpdateTime

        return False

    def stateStalled(self, msgin):
        self.decodeHeader(msgin)
        if self._SystemMode:
            message = msgin.readUint8('message')
            self.log.debug("_CurrentReceivedNumber:%d (mode:%s) %d [%d/%d/%d]" % (self._CurrentReceivedNumber, str(self._SystemMode), message, msgin.sizeData(),  msgin.sizeRead(), msgin.needRead()))
            if message == CLFECOMMON.SYSTEM_SYNC_CODE:
                self.log.debug("stalled->synchronize")
                self._ConnectionState = TConnectionState.Synchronize
                self.receiveSystemSync(msgin)
                return True
            elif message == CLFECOMMON.SYSTEM_PROBE_CODE:
                self.log.debug("stalled->probe")
                self._ConnectionState = TConnectionState.Probe
                self.receiveSystemProbe(msgin)
            elif message == CLFECOMMON.SYSTEM_STALLED_CODE:
                self.receiveSystemStalled(msgin)
            elif message == CLFECOMMON.SYSTEM_SERVER_DOWN_CODE:
                self.disconnect()
                self.log.warning("BACK-END DOWN")
                return False
            else:
                self.log.warning("CNET: received system %d in state Stalled" % message)
            self.log.debug("_CurrentReceivedNumber:%d (mode:%s) %d [%d/%d/%d] '%s'" % (self._CurrentReceivedNumber, str(self._SystemMode), message, msgin.sizeData(),  msgin.sizeRead(), msgin.needRead(), msgin.showLastData()))
        else:
            self.log.warning("received normal in state Stalled")
        return False

    def stateQuit(self, msgin):
        self.decodeHeader(msgin)
        if self._SystemMode:
            message = msgin.readUint8('message')
            self.log.debug("_CurrentReceivedNumber:%d (mode:%s) %d [%d/%d/%d]" % (self._CurrentReceivedNumber, str(self._SystemMode), message, msgin.sizeData(),  msgin.sizeRead(), msgin.needRead()))
            if message == CLFECOMMON.SYSTEM_SYNC_CODE:
                self.log.debug("quit->synchronize")
                self._ConnectionState = TConnectionState.Synchronize
                self.receiveSystemSync(msgin)
                return True
            elif message == CLFECOMMON.SYSTEM_SERVER_DOWN_CODE:
                self.disconnect()
                self.log.warning("BACK-END DOWN")
                return False
            elif message == CLFECOMMON.SYSTEM_ACK_QUIT_CODE:
                self.receiveSystemAckQuit(msgin)
            else:
                self.log.warning("CNET: received system %d in state Quit" % message)
            self.log.debug("_CurrentReceivedNumber:%d (mode:%s) %d [%d/%d/%d] '%s'" % (self._CurrentReceivedNumber, str(self._SystemMode), message, msgin.sizeData(),  msgin.sizeRead(), msgin.needRead(), msgin.showLastData()))
        else:
            self.log.warning("received normal in state Quit")
        if not self._ReceivedAckQuit and (self._UpdateTime - self._LatestQuitTime > 100):
            self.sendSystemQuit()
            self._LatestQuitTime = self._UpdateTime
        return False

    def update(self):
        # khanat-opennel-code/code/ryzom/client/src/network_connection.cpp # bool CNetworkConnection::update()
        self._UpdateTime = int(time.clock_gettime(1)*1000)
        self._ReceivedSync = False

        if not self._sock:
            return False

        # TODO - REMOVE this counter (just to stop loop)
        counterLoop = 0

        stateBroke = True
        while stateBroke:
            buffer, addr = self.buildStream()
            msgin = BitStream()
            msgin.fromBytes(buffer)
            self.log.debug("received message: %s" % msgin.showAllData())
            if self._ConnectionState == TConnectionState.Login:
                self.log.debug("state:Login")
                stateBroke = self.stateLogin(msgin)
            elif self._ConnectionState == TConnectionState.Synchronize:
                self.log.debug("state:Synchronize")
                stateBroke = self.stateSynchronize(msgin)
            elif self._ConnectionState == TConnectionState.Connected:
                self.log.debug("state:Connected")
                stateBroke = self.stateConnected(msgin)
            elif self._ConnectionState == TConnectionState.Probe:
                self.log.debug("state:Probe")
                stateBroke = self.stateProbe(msgin)
            elif self._ConnectionState == TConnectionState.Stalled:
                self.log.debug("state:Stalled")
                stateBroke = self.stateStalled(msgin)
            elif self._ConnectionState == TConnectionState.Quit:
                self.log.debug("state:Quit")
                stateBroke = self.stateQuit(msgin)
            else:
                stateBroke = False
            self.log.debug("message decoded: %s" % msgin.showAllData())
            counterLoop += 1
            if counterLoop > 10:
                break

    def send(self):
        if self._ConnectionState == TConnectionState.Connected and self._LastSendTime > 100:
            self.sendNormalMessage()

    def EmulateFirst(self, msgRawXml, databaseRawXml):
        self.msgXml = ET.fromstring(msgRawXml)
        #ET.dump(msgXml)
        self.databaseXml = ET.fromstring(databaseRawXml)
        #ET.dump(databaseXml)

        self._MsgXmlMD5 = getTextMD5(msgRawXml)
        self._DatabaseXmlMD5 = getTextMD5(databaseRawXml)

        self.decodeImpulse.loadMsg(self.msgXml)
        self.decodeImpulse.loadDatabase(self.databaseXml)
        self.connect()
        self.log.info("Client Login")
        self.sendSystemLogin()

        self.log.info("Receive Message")
        i = 0
        for _ in range(0, 50):
        #while True:
            self.log.debug("%s [%s: %d] %s" % ("*" * 40, "Loop", i,"*" * 40))
            self.update()
            self.send()
            i += 1

        self.log.info("Client Quit")
        self.sendSystemQuit()


class ClientKhanat:
    def __init__(self,
                 khanat_host,
                 khanat_port_login = 40916,
                 khanat_port_frontend = 47851,
                 login="tester",
                 password="tester",
                 clientApp="Lirria",
                 LanguageCode="fr",
                 url="/login/r2_login.php",
                 suffix = None,
                 download_patch = False,
                 show_patch_detail=False,
                 size_buffer_file=1024):
        self.log = logging.getLogger('myLogger')

        if suffix is None:
            suffix = str(random.randrange(1, 9999))
            self.log.debug("suffix : %s" % suffix)

        self.download_patch = download_patch
        self.show_patch_detail = show_patch_detail
        self.khanat_host = khanat_host
        self.khanat_port_login = khanat_port_login
        self.khanat_port_frontend = khanat_port_frontend
        self.login = login + suffix
        self.password = password
        self.clientApp = clientApp
        self.LanguageCode = LanguageCode
        self.url = url
        self.cookie, self.fsaddr, self.ringmainurl, self.fartp, self.stat, self.r2serverversion, self.r2backuppatchurl, self.r2patchurl = None, None, None, None, None, None, None, None
        self.tempdir = tempfile.TemporaryDirectory(".khanat")
        self.log.debug("Temporary directory:%s" % self.tempdir)
        self.khanat_idx = CPersistentDataRecord(self.log)
        self.UserAddr, self.UserKey, self.UserId = None, None, None
        self.clientNetworkConnection = ClientNetworkConnection(self.khanat_host, self.khanat_port_frontend)
        self.size_buffer_file = size_buffer_file
        self.cFileContainer = CFileContainer()

    def createAccount(self):
        conn = http.client.HTTPConnection(host=self.khanat_host, port=self.khanat_port_login)
        cmd = "/ams/index.php?page=register"
        headers = {'Accept':  'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
                            'Accept-Language' : 'en-US',
                            'Connection': 'keep-alive',
                            'DNT': '1',
                            'Cookie': 'PHPSESSID=lsoumn9f0ljgm3vo3hgjdead03',
                            'Host': self.khanat_host+':'+ str(self.khanat_port_login),
                            'Referer': 'http://' + self.khanat_host+':'+ str(self.khanat_port_login) + '/ams/index.php?page=register',
                            'Upgrade-Insecure-Requests': '1',
                            'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; rv:6.0) Gecko/20100101 Firefox/6.0',
                            'Content-Type': 'application/x-www-form-urlencoded'}

        headers = {'Content-Type': 'application/x-www-form-urlencoded'}
        params = urllib.parse.urlencode({'Username': self.login, 'Password': self.password, 'ConfirmPass': self.password,  'Email': self.login+'@khaganat.net',  'TaC': 'on',  'function': 'add_user'})
        conn.request("POST", cmd,  params, headers)
        response = conn.getresponse()
        if ( int(response.status) == 302 ):
            conn.close()
            self.log.info("Account : %s" % self.login)
            return
        elif ( int(response.status) != 200 ):
            self.log.error("Impossible to create account (return code:" + str(response.status) + ")")
            sys.exit(2)
        ret = response.read()
        conn.close()

        ret2 = ret.decode()
        try:
            state, comment = ret2.split(":", 2)
        except:
            state = 1
            comment = ""
        if int(state) != 1:
            self.log.error("Impossible to create account (state:" + state + ", comment:" + comment.strip() + ")")
            sys.exit(2)

        errordetected = False
        for line in ret2.split('\n'):
            m = re.search("(<strong>(?P<type>.*) Error</strong> )(?P<comment>[^.]+)",  line)
            if m:
                if m.group('comment') == 'Username ' + self.login + ' is in use':
                    continue
                if m.group('comment') == 'Email is in use':
                    continue
                self.log.error('Impossible to create account: field:%s (%s)' % (m.group('type'),  m.group('comment')))
                errordetected = True
        if errordetected:
            sys.exit(2)
        self.log.info("Reuse account : %s" % self.login)

    def connectR2(self):
        conn = http.client.HTTPConnection(host=self.khanat_host, port=self.khanat_port_login)
        cmd = self.url + "?cmd=ask&cp=2&login=" + self.login + "&lg=" + self.LanguageCode
        conn.request("GET", cmd)
        response = conn.getresponse()
        if ( int(response.status) != 200 ):
            self.log.error("Impossible to get salt (return code:" + str(response.status) + ")")
            sys.exit(2)
        ret = response.read()
        conn.close()

        try:
            state, salt = ret.decode().split(":", 1)
        except UnicodeDecodeError:
            try:
                state, salt = ret.decode(encoding='cp1252').split(":", 1)
            except UnicodeDecodeError:
                self.log.error("Impossible to read output login")
                sys.exit(2)
        if int(state) != 1:
            self.log.error("Impossible to get salt (state:" + state + ")")

        cryptedPassword = crypt.crypt(self.password, salt)

        conn = http.client.HTTPConnection(host=self.khanat_host, port=self.khanat_port_login)
        cmd = self.url + "?cmd=login&login=" + self.login + "&password=" + cryptedPassword + "&clientApplication=" + self.clientApp + "&cp=2" + "&lg=" + self.LanguageCode
        conn.request("GET", cmd)
        response = conn.getresponse()
        self.log.debug("%s %s" %(response.status,  response.reason))
        ret = response.read()
        self.log.debug(ret)
        try:
            line = ret.decode().split('\n')
        except UnicodeDecodeError:
            try:
                line = ret.decode(encoding='cp1252').split('\n')
            except UnicodeDecodeError:
                self.log.error("Impossible to read output login")
                sys.exit(2)

        self.log.debug(line[0])
        self.log.debug("line 0 '%s'" % line[0])
        self.log.debug("line 1 '%s'" % line[1])
        try:
            state, self.cookie, self.fsaddr, self.ringmainurl, self.fartp, self.stat = line[0].split("#", 6)
        except:
            try:
                state, self.cookie, self.fsaddr, self.ringmainurl, self.fartp = line[0].split("#", 5)
                self.stat = 0
            except:
                state, error = line[0].split(":", 1)

        if int(state) != 1:
            self.log.error(error)
            sys.exit(2)

        self.r2serverversion, self.r2backuppatchurl, self.r2patchurl = line[1].split("#")
        self.log.debug("%s %s %s %s %s %s %s %s %s" % (state, self.cookie, self.fsaddr, self.ringmainurl, self.fartp, self.stat, self.r2serverversion, self.r2backuppatchurl, self.r2patchurl))
        self.UserAddr, self.UserKey, self.UserId = [ int(x,  16) for x in self.cookie.split('|') ]

        conn.close()
        self.log.info("Login Ok")
        self.clientNetworkConnection.cookiesInit(self.UserAddr, self.UserKey, self.UserId)

    def downloadFileUrl(self, source, dest):
        self.log.info("Download %s (destination:%s)" % (source, dest))
        with urllib.request.urlopen(source) as conn :
            header = conn.getheaders()
            file_size = 0
            for key, value in header:
                if key == 'Content-Length':
                    file_size = int(value)
                    break
            self.log.debug("size:%d", file_size)
            file_size_dl = 0
            block_size = self.size_buffer_file # 1024

            with open(dest, 'wb') as fp:
                while True:
                    buffer = conn.read(block_size)
                    if not buffer:
                        break
                    file_size_dl += len(buffer)
                    fp.write(buffer)
                    self.log.debug("Download %s %10d [%6.2f%%]" % (source, file_size_dl, file_size_dl * 100. / file_size))
                fp.close()
            self.log.debug("Downloaded %s (%d)" % (source, file_size))

    def getServerFile(self, name, bZipped = False, specifyDestName = None):
        srcName = name
        if specifyDestName:
            dstName = specifyDestName
        else:
            dstName = os.path.basename(name)
        if bZipped:
            srcName += ".ngz"
            dstName += ".ngz"
        self.log.info("Download %s (destination:%s, zip:%d)" % (srcName, dstName, bZipped))
        dstName = os.path.join(self.tempdir.name, dstName)
        self.downloadFileUrl( 'http://' + self.r2patchurl + '/' + srcName, dstName)
        return dstName

    def downloadAllPatch(self):
        # TODO - check where client search file to download
        for file in self.khanat_idx.CBNPFile:
            tmp = self.getServerFile("%05d/%s.lzma" % (int(self.r2serverversion), file.FileName), False, "")
            with lzma.open(tmp) as fin:
                dstName = os.path.join(self.tempdir.name, file.FileName)
                with open(dstName, "wb") as fout:
                    data = fin.read()
                    fout.write(data)
                self.log.info("%s" % dstName)
            os.remove(tmp)
        # khanat-opennel-code/code/ryzom/client/src/login_patch.cpp # void CCheckThread::run ()
        FilesToPatch = []
        for file in self.khanat_idx.CBNPFile:
            FilesToPatch.append(file)
        # Here we got all the files to patch in FilesToPatch and all the versions that must be obtained Now we have to get the optional categories
        OptionalCat = []
        for category in self.khanat_idx.Categories:
            if category._IsOptional:
                for file in category._Files:
                    bAdded = False
                    for file2 in FilesToPatch:
                        if file2 == file:
                            OptionalCat.append(category._Name)
                            bAdded = True
                            break
                    if bAdded:
                        break
        # For all categories that required an optional category if the cat required is present the category that reference it must be present
        for category in self.khanat_idx.Categories:
            if category._IsOptional and not len(category._CatRequired) == 0:
                bFound = False
                for cat in OptionalCat:
                    if category._Name == cat:
                        bFound = True
                        break
                if bFound:
                    for cat in OptionalCat:
                        if category._CatRequired == cat:
                            OptionalCat.append(category._Name)
                            break
        # Delete categories optional cat that are hidden
        for category in self.khanat_idx.Categories:
            if category._IsOptional and category._Hidden:
                for cat in OptionalCat:
                    if category._Name == cat:
                        OptionalCat.remove(category._Name)
                        break
        # Get all extract to category and check files inside the bnp with real files
        for category in self.khanat_idx.Categories:
            if len(category._UnpackTo) != 0:
                for file in category._Files:
                    # TODO
                    # readHeader()
                    pass

    def DownloadMinimum(self):
        self.log.debug("-" * 80)
        for file in self.khanat_idx.CBNPFile:
            if file.FileName != "kh_server.bnp":
                continue
            tmp = self.getServerFile("%05d/%s.lzma" % (int(self.r2serverversion), file.FileName), False, "")
            with lzma.open(tmp) as fin:
                dstName = os.path.join(self.tempdir.name, file.FileName)
                with open(dstName, "wb") as fout:
                    data = fin.read()
                    fout.write(data)
                self.log.info("%s" % dstName)
            os.remove(tmp)

    def Emulate(self):
        self.createAccount()
        self.connectR2()
        # download patch
        self.ryzomidx = self.getServerFile("%05d/ryzom_%05d.idx" % (int(self.r2serverversion), int(self.r2serverversion)), False, "")
        self.khanat_idx.readFromBinFile(self.ryzomidx)
        self.khanat_idx.CProductDescriptionForClient_apply()
        # Show detail patch
        if self.show_patch_detail:
            self.khanat_idx.decrypt_token()
            self.khanat_idx.show()
        # Todo analyze patch and download if necessary or update if incremental - see category
        # Download all file in patch - login_patch.cpp:2578 # void CPatchThread::processFile (CPatchManager::SFileToPatch &rFTP)
        if self.download_patch:
            self.downloadAllPatch()
        else:
            self.DownloadMinimum()
        self.cFileContainer = CFileContainer()
        self.cFileContainer.addSearchPath(self.tempdir.name)
        msgRawXml = self.cFileContainer.getdata("msg.xml").decode()
        databaseRawXml = self.cFileContainer.getdata("database.xml").decode()
        self.clientNetworkConnection.EmulateFirst(msgRawXml, databaseRawXml)


def main():
    FORMAT = '%(asctime)-15s %(levelname)s %(filename)s:%(lineno)d %(message)s'
    logging.basicConfig(format=FORMAT)
    log = logging.getLogger('myLogger')

    parser = argparse.ArgumentParser()
    parser.add_argument("--khanat-host", help="khanat host to auhtenticate",  default='localhost')
    parser.add_argument("--suffix", help="define suffix")
    parser.add_argument("-d", "--debug", help="show debug message", action='store_true')
    parser.add_argument("-p", "--download-patch", help="show debug message", action='store_true')
    parser.add_argument("-s", "--show-patch-detail", help="show debug message", action='store_true')
    parser.add_argument("--size-buffer-file", help="size buffer to download file", type=int, default=1024)
    parser.add_argument("--khanat-port-login", help="port http login", type=int, default=40916)
    parser.add_argument("--khanat-port-frontend", help="port UDP frontend", type=int, default=47851)

    args = parser.parse_args()

    if args.debug:
        level = logging.getLevelName('DEBUG')
    else:
        level = logging.getLevelName('INFO')
    log.setLevel(level)

    client = ClientKhanat(args.khanat_host, khanat_port_login=args.khanat_port_login, khanat_port_frontend=args.khanat_port_frontend, suffix=args.suffix, download_patch=args.download_patch, show_patch_detail=args.show_patch_detail, size_buffer_file=args.size_buffer_file)
    client.Emulate()
    log.info("End")

if __name__ == "__main__":
    #TestBitStream()
    #TestCBitSet()
    main()