Changed: #929 Max shape export runs all the way through now.

Added: #929, #1082 Interface build project.

code/nel/tools/build_gamedata/configuration/scripts.py

@@ -181,6 +181,20 @@ def findFiles(log, dir_where, dir_sub, file_ext):
 				printLog(log, "findFiles: file not dir or file?!" + filePath)
 	return result
 
+def findFilesNoSubdir(log, dir_where, file_ext):
+	result = [ ]
+	files = os.listdir(dir_where)
+	len_file_ext = len(file_ext)
+	for fileName in files:
+		if fileName != ".svn" and fileName != "*.*":
+			fileFull = dir_where + "/" + fileName
+			if os.path.isfile(fileFull):
+				if fileName[-len_file_ext:].lower() == file_ext.lower():
+					result += [ fileName ]
+			elif not os.path.isdir(fileFull):
+				printLog(log, "findFilesNoSubdir: file not dir or file?!" + filePath)
+	return result
+
 def findFile(log, dir_where, file_name):
 	files = os.listdir(dir_where)
 	for fileName in files:

code/nel/tools/build_gamedata/configuration/tools.py

@@ -49,6 +49,8 @@ LigoExportTimeout = 3600000
 LigoBuildTimeout = 1800000
 PacsPrimExportTimeout = 600000
 
+MaxShapeExportTimeout = 300000 # 5min
+
 # *** TOOLS CONFIGURATION ***
 
 TgaToDdsTool = "tga2dds"

code/nel/tools/build_gamedata/processes/interface/0_setup.py

@@ -45,7 +45,8 @@ printLog(log, "")
 
 # Setup source directories
 printLog(log, ">>> Setup source directories <<<")
-for dir in InterfaceSourceDirectories:
+for dirs in InterfaceSourceDirectories:
+	for dir in dirs:
 		mkPath(log, DatabaseDirectory + "/" + dir)
 for dir in InterfaceDxtcSourceDirectories:
 	mkPath(log, DatabaseDirectory + "/" + dir)

code/nel/tools/build_gamedata/processes/interface/1_export.py

@@ -50,12 +50,14 @@ printLog(log, "")
 # For each interface directory
 printLog(log, ">>> Export interface <<<")
 mkPath(log, ExportBuildDirectory + "/" + InterfaceExportDirectory)
-for dir in InterfaceSourceDirectories:
+for dirs in InterfaceSourceDirectories:
-	mkPath(log, DatabaseDirectory + "/" + dir)
+	niouname = dirs[0].replace("/", "_")
-	niouname = dir.replace("/", "_")
 	newpath = ExportBuildDirectory + "/" + InterfaceExportDirectory + "/" + niouname
 	mkPath(log, newpath)
+	for dir in dirs:
+		mkPath(log, DatabaseDirectory + "/" + dir)
 		copyFilesExtNoTreeIfNeeded(log, DatabaseDirectory + "/" + dir, newpath, ".tga")
+		copyFilesExtNoTreeIfNeeded(log, DatabaseDirectory + "/" + dir, newpath, ".png")
 printLog(log, "")
 
 # For each interface directory to compress in one DXTC
@@ -64,6 +66,7 @@ mkPath(log, ExportBuildDirectory + "/" + InterfaceDxtcExportDirectory)
 for dir in InterfaceDxtcSourceDirectories:
 	mkPath(log, DatabaseDirectory + "/" + dir)
 	copyFilesExtNoTreeIfNeeded(log, DatabaseDirectory + "/" + dir, ExportBuildDirectory + "/" + InterfaceDxtcExportDirectory, ".tga")
+	copyFilesExtNoTreeIfNeeded(log, DatabaseDirectory + "/" + dir, ExportBuildDirectory + "/" + InterfaceDxtcExportDirectory, ".png")
 printLog(log, "")
 
 # For each interface fullscreen directory compress independently all in dds
@@ -80,6 +83,12 @@ else:
 			destFile = ExportBuildDirectory + "/" + InterfaceFullscreenExportDirectory + "/" + os.path.basename(file)[0:-len(".tga")] + ".dds"
 			if needUpdateLogRemoveDest(log, sourceFile, destFile):
 				subprocess.call([ TgaToDds, sourceFile, "-o", destFile, "-a", "5" ])
+		files = findFiles(log, DatabaseDirectory + "/" + dir, "", ".png")
+		for file in files:
+			sourceFile = DatabaseDirectory + "/" + dir + "/" + file
+			destFile = ExportBuildDirectory + "/" + InterfaceFullscreenExportDirectory + "/" + os.path.basename(file)[0:-len(".png")] + ".dds"
+			if needUpdateLogRemoveDest(log, sourceFile, destFile):
+				subprocess.call([ TgaToDds, sourceFile, "-o", destFile, "-a", "5" ])
 printLog(log, "")
 
 # For each interface 3d directory
@@ -88,6 +97,7 @@ mkPath(log, ExportBuildDirectory + "/" + Interface3DExportDirectory)
 for dir in Interface3DSourceDirectories:
 	mkPath(log, DatabaseDirectory + "/" + dir)
 	copyFilesExtNoTreeIfNeeded(log, DatabaseDirectory + "/" + dir, ExportBuildDirectory + "/" + Interface3DExportDirectory, ".tga")
+	copyFilesExtNoTreeIfNeeded(log, DatabaseDirectory + "/" + dir, ExportBuildDirectory + "/" + Interface3DExportDirectory, ".png")
 printLog(log, "")
 
 log.close()

code/nel/tools/build_gamedata/processes/shape/1_export.py

@@ -27,16 +27,23 @@
 import time, sys, os, shutil, subprocess, distutils.dir_util
 sys.path.append("../../configuration")
 
-if os.path.isfile("log.log"):
+if os.path.isfile("temp_log.log"):
-	os.remove("log.log")
+	os.remove("temp_log.log")
-log = open("log.log", "w")
+log = open("temp_log.log", "w")
 from scripts import *
 from buildsite import *
 from process import *
 from tools import *
 from directories import *
 
+# Todo: make a tool to verify the files :)
+
+def getTagFileName(filePath):
+	return os.path.split(filePath)[1] + ".tag"
+
 def hackBigTree():
+	return 0
+	if EcosystemName == "jungle":
 		# FO_S2_big_tree is corrupt on first export...
 		outDirTag = ExportBuildDirectory + "/" + ShapeTagExportDirectory
 		outDirWithCoarse = ExportBuildDirectory + "/" + ShapeWithCoarseMeshExportDirectory
@@ -60,6 +67,7 @@ def hackBigTree():
 		else:
 			return 0
 
+
 printLog(log, "")
 printLog(log, "-------")
 printLog(log, "--- Export shape")
@@ -78,7 +86,7 @@ if BuildQuality == 0:
 if MaxAvailable:
 	# Find tools
 	Max = findMax(log, MaxDirectory, MaxExecutable)
-	# ExecTimeout = findTool(log, ToolDirectories, ExecTimeoutTool, ToolSuffix)
+	ExecTimeout = findTool(log, ToolDirectories, ExecTimeoutTool, ToolSuffix)
 	printLog(log, "")
 	
 	# Export shape 3dsmax
@@ -108,13 +116,20 @@ if MaxAvailable:
 		os.remove(scriptDst)
 	for dir in ShapeSourceDirectories:
 		tagDiff = 1
+		secondTry = 1
 		shapeSourceDir = DatabaseDirectory + "/" + dir
 		mkPath(log, shapeSourceDir)
+		maxFiles = findFilesNoSubdir(log, shapeSourceDir, ".max")
+		for maxFile in maxFiles:
+			maxFilePath = shapeSourceDir + "/" + maxFile
+			tagFilePath = outDirTag + "/" + getTagFileName(maxFilePath)
+			if (needUpdate(log, maxFilePath, tagFilePath)):
 				sSrc = open(scriptSrc, "r")
 				sDst = open(scriptDst, "w")
 				for line in sSrc:
 					newline = line.replace("output_logfile", logFile)
-			newline = newline.replace("shape_source_directory", shapeSourceDir)
+					# newline = newline.replace("shape_source_directory", shapeSourceDir)
+					newline = newline.replace("shape_max_file_path", maxFilePath)
 					newline = newline.replace("output_directory_tag", outDirTag)
 					newline = newline.replace("output_directory_without_coarse_mesh", outDirWithoutCoarse)
 					newline = newline.replace("output_directory_with_coarse_mesh", outDirWithCoarse)
@@ -129,16 +144,38 @@ if MaxAvailable:
 					sDst.write(newline)
 				sSrc.close()
 				sDst.close()
-		while tagDiff > 0:
+				retriesLeft = 5
-			printLog(log, "MAXSCRIPT " + scriptDst)
+				while retriesLeft > 0:
-			subprocess.call([ Max, "-U", "MAXScript", "shape_export.ms", "-q", "-mi", "-vn" ])
+					printLog(log, "MAXSCRIPT " + scriptDst + "; " + maxFilePath)
-			tagList = findFiles(log, outDirTag, "", ".tag")
+					subprocess.call([ ExecTimeout, str(MaxShapeExportTimeout), Max, "-U", "MAXScript", "shape_export.ms", "-q", "-mi", "-vn" ])
-			newTagLen = len(tagList)
+					lSrc = open(logFile, "r")
-			tagDiff = newTagLen - tagLen
+					for line in lSrc:
-			tagLen = newTagLen
+						if (len(line) > 0):
-			printLog(log, "Exported " + str(tagDiff) + " .max files!")
+							printLog(log, line.strip())
-			tagDiff += hackBigTree() # force rerun also when big tree deleted
+					lSrc.close()
+					os.remove(logFile)
+					if (os.path.exists(tagFilePath)):
+						printLog(log, "OK " + maxFilePath)
+						retriesLeft = 0
+					else:
+						printLog(log, "FAIL " + maxFilePath)
+					retriesLeft = retriesLeft - 1
 				os.remove(scriptDst)
+			else:
+				printLog(log, "SKIP " + maxFilePath)
+#while tagDiff > 0:
+#	printLog(log, "MAXSCRIPT " + scriptDst)
+#	subprocess.call([ Max, "-U", "MAXScript", "shape_export.ms", "-q", "-mi", "-vn" ])
+#	tagList = findFiles(log, outDirTag, "", ".tag")
+#	newTagLen = len(tagList)
+#	tagDiff = newTagLen - tagLen
+#	tagLen = newTagLen
+#	printLog(log, "Exported " + str(tagDiff) + " .max files!")
+#	if not tagDiff > 0:
+#		tagDiff += hackBigTree() # force rerun also when big tree deleted
+#		if not tagDiff > 0:
+#			tagDiff += secondTry
+#			secondTry = 0
 	
 	# Export clod 3dsmax
 	printLog(log, ">>> Export character lod shape files (.clod) from Max <<<")
@@ -156,6 +193,6 @@ if MaxAvailable:
 printLog(log, "")
 
 log.close()
-
+shutil.move("temp_log.log", "log.log")
 
 # end of file

code/nel/tools/build_gamedata/processes/shape/maxscript/shape_export.ms

@@ -245,18 +245,9 @@ fn goShapeExport =
 (
 	try
 	(
-		-- Get files in the shape_source_directory
+		-- Get filepath
-		files = getFiles "shape_source_directory/*.max" 
+		filePath = "shape_max_file_path"
 		
-		-- Sort files
-		sort files
-
-		-- No file ?
-		if files.count != 0 then
-		(
-			-- For each files
-			for i = 1 to files.count do
-			(
 		try
 		(
 			-- Delete lod files
@@ -272,14 +263,11 @@ fn goShapeExport =
 			resetMAXFile #noprompt
 			
 			-- Get the tag file name
-					tag = ("output_directory_tag/"+(getFilenameFile files[i])+(getFilenameType files[i])+".tag")
+			tag = ("output_directory_tag/" + (getFilenameFile filePath) + (getFilenameType filePath) + ".tag")
 			
-					-- Compare date with the tag file
-					if (NeLTestFileDate tag files[i]) == true then
-					(
 			-- Open the max project
-						nlerror ("Scanning file "+files[i]+" ...")
+			nlerror ("Scanning file " + filePath + " ...")
-						if loadMaxFile files[i] quiet:true == true then
+			if loadMaxFile filePath quiet:true == true then
 			(
 				-- Unhide category
 				unhidecategory()
@@ -381,7 +369,7 @@ fn goShapeExport =
 										output = ("output_directory_without_coarse_mesh/"+(node.name)+".shape")
 									
 									-- Compare file date
-												if (NeLTestFileDate output files[i]) == true then
+									if (NeLTestFileDate output filePath) == true then
 									(
 										try
 										(
@@ -394,7 +382,7 @@ fn goShapeExport =
 											else
 											(
 												-- Error
-															nlerror ("ERROR exporting shape "+node.name+" in file "+files[i])
+												nlerror ("ERROR exporting shape "+node.name+" in file "+filePath)
 												tagThisFile = false
 												return 0
 											)
@@ -402,7 +390,7 @@ fn goShapeExport =
 										catch 
 										(
 											-- Error
-														nlerror ("ERROR fatal error exporting shape "+node.name+" in file "+files[i])
+											nlerror ("ERROR fatal error exporting shape "+node.name+" in file "+filePath)
 											tagThisFile = false
 											return 0
 										)
@@ -461,51 +449,67 @@ fn goShapeExport =
 				if exported == 0 then
 				(
 					-- Error
-								nlerror ("WARNING no shape exported from the file "+files[i])
+					nlerror ("WARNING no shape exported from the file "+filePath)
 				)
 			)
 			else
 			(
 				-- Error
-							nlerror ("ERROR exporting shape: can't open the file "+files[i])
+				nlerror ("ERROR exporting shape: can't open the file "+filePath)
-						)
-					)
-					else
-					(
-						-- Error
-						nlerror ("SKIPPED BY TAG "+files[i])
 			)
 			
+			try
+			(
 				gc ()
+			)
+			catch
+			(
+				nlerror ("ERROR gc " + getCurrentException())
+			)
 			
+			try
+			(
 				-- Reset 3dsmax
 				resetMAXFile #noprompt
 			)
 			catch
 			(
-					-- Error
+				nlerror ("ERROR resetMAXFile " + getCurrentException())
-					nlerror ("ERROR fatal error exporting shape in file " + files[i])
-					-- return 0
 			)
-			)
+			
-		)
+			try
-		else
 			(
-			nlerror ("WARNING no max file in folder shape_source_directory")
+				gc ()
+			)
+			catch
+			(
+				nlerror ("ERROR gc " + getCurrentException())
 			)
 		)
 		catch 
 		(
 			-- Error
-		nlerror ("ERROR fatal error exporting shape in folder shape_source_directory")
+			nlerror ("ERROR fatal error exporting shape in file " + filePath)
+			return 0
+		)
+	)
+	catch 
+	(
+		-- Error
+		nlerror ("ERROR fatal error exporting shape in file")
 		return 0
 	)
 )
 
-goShapeExport()
+undo off
+(
+	goShapeExport()
+)
 
 nlerror ("BYE")
 
 quitMAX #noPrompt
 quitMAX () #noPrompt
 
+quitMAX #noPrompt
+quitMAX () #noPrompt

code/ryzom/tools/build_gamedata/workspace/common/interface/directories.py

@@ -0,0 +1,86 @@
+#!/usr/bin/python
+# 
+# \file directories.py
+# \brief Directories configuration
+# \date 2010-08-27 17:13GMT
+# \author Jan Boon (Kaetemi)
+# \date 2001-2005
+# \author Nevrax
+# Python port of game data build pipeline.
+# Directories configuration.
+# 
+# NeL - MMORPG Framework <http://dev.ryzom.com/projects/nel/>
+# Copyright (C) 2010  Winch Gate Property Limited
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# 
+
+# *** COMMON PATH ***
+
+CommonPath = "common/interface"
+
+# *** SOURCE DIRECTORIES IN THE DATABASE ***
+
+#Interface directories
+InterfaceSourceDirectories = [ ]
+InterfaceSourceDirectories += [ [ "interfaces/v3" ] + [ "interfaces/ring_interface" ] ]
+InterfaceSourceDirectories += [ [ "interfaces/v3_outgame/ui" ] ]
+InterfaceSourceDirectories += [ [ "interfaces/v3_login" ] ]
+
+InterfaceDxtcSourceDirectories = [ ]
+InterfaceDxtcSourceDirectories += [ "interfaces/v3_bricks" ]
+InterfaceDxtcSourceDirectories += [ "interfaces/v3_items" ]
+InterfaceDxtcSourceDirectories += [ "interfaces/v3_dxtc_misc" ]
+
+InterfaceFullscreenSourceDirectories = [ ]
+InterfaceFullscreenSourceDirectories += [ "interfaces/v3_fullscreen" ]
+InterfaceFullscreenSourceDirectories += [ "interfaces/v3_outgame/fullscreen" ]
+InterfaceFullscreenSourceDirectories += [ "interfaces/v3_doc/graph/abilities_items" ]
+InterfaceFullscreenSourceDirectories += [ "interfaces/v3_doc/graph/buy_sell" ]
+InterfaceFullscreenSourceDirectories += [ "interfaces/v3_doc/graph/camera_character" ]
+InterfaceFullscreenSourceDirectories += [ "interfaces/v3_doc/graph/create_perso" ]
+InterfaceFullscreenSourceDirectories += [ "interfaces/v3_doc/graph/fight" ]
+InterfaceFullscreenSourceDirectories += [ "interfaces/v3_doc/graph/MatisTown" ]
+InterfaceFullscreenSourceDirectories += [ "interfaces/v3_doc/graph/spell" ]
+InterfaceFullscreenSourceDirectories += [ "interfaces/v3_doc/graph/talk_bot" ]
+InterfaceFullscreenSourceDirectories += [ "interfaces/v3_doc/graph/abilities_items" ]
+InterfaceFullscreenSourceDirectories += [ "interfaces/v3_quick_help/graph" ]
+
+Interface3DSourceDirectories = [ ]
+Interface3DSourceDirectories += [ "interfaces/v3_outgame/3d" ]
+Interface3DSourceDirectories += [ "interfaces/v3_doc/htm" ]
+Interface3DSourceDirectories += [ "interfaces/v3_doc" ]
+Interface3DSourceDirectories += [ "interfaces/v3_quick_help" ]
+
+
+# *** EXPORT DIRECTORIES FOR THE BUILD PIPELINE ***
+
+# Interface directories
+InterfaceExportDirectory = CommonPath + "/interface_export"
+InterfaceDxtcExportDirectory = CommonPath + "/interface_dxtc_export"
+InterfaceFullscreenExportDirectory = CommonPath + "/interface_fullscreen_export"
+Interface3DExportDirectory = CommonPath + "/interface_3d_export"
+
+
+# *** BUILD DIRECTORIES FOR THE BUILD PIPELINE ***
+
+# Interface directories
+InterfaceBuildDirectory = CommonPath + "/interface_build"
+InterfaceDxtcBuildDirectory = CommonPath + "/interface_dxtc_build"
+
+
+# *** INSTALL DIRECTORIES IN THE CLIENT DATA ***
+
+# Lightmap directory
+InterfaceClientDirectory = "interfaces"

code/ryzom/tools/build_gamedata/workspace/common/interface/process.py

@@ -0,0 +1,35 @@
+#!/usr/bin/python
+# 
+# \file config.py
+# \brief Process configuration
+# \date 2010-08-27 17:02GMT
+# \author Jan Boon (Kaetemi)
+# Python port of game data build pipeline.
+# Process configuration.
+# 
+# NeL - MMORPG Framework <http://dev.ryzom.com/projects/nel/>
+# Copyright (C) 2010  Winch Gate Property Limited
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# 
+
+# *** PROCESS CONFIGURATION ***
+
+# *** PROCESS CONFIG ***
+ProcessToComplete = [ ]
+ProcessToComplete += [ "interface" ]
+
+# *** MAPS OPTIONS ***
+
+ReduceBitmapFactor = 0

code/ryzom/tools/build_gamedata/workspace/projects.py

@@ -26,6 +26,7 @@
 
 
 ProjectsToProcess = [ ]
+ProjectsToProcess += [ "common/interface" ]
 ProjectsToProcess += [ "ecosystems/jungle" ]
 ProjectsToProcess += [ "continents/newbieland" ]