You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 
 
 

493 lines
22 KiB

from multiprocessing import Process, Pipe
import time
import os
import shutil
import subprocess
import sys
import stat
import traceback
import psutil
import json
from PyModules.BuildLogTail import BuildLogTail, PrintLogASAP
from PyModules.PrepareParallelProj import PrepareParallelProj4Win64, PrepareParallelProj4Android, PrepareParallelProj4IOS
import tools
_IsAndroid = False
_IsIOS = False
_IsWin64 = False
# 参数配置举例
#UnityPath = 'C:/Program Files/Unity/Hub/Editor/2018.4.19f1/Editor/Unity.exe'
#JAVAPath = 'C:/Program Files/Unity/Hub/Editor/2018.4.19f1/Editor/Data/PlaybackEngines/AndroidPlayer/Tools/OpenJDK/Windows/bin/java.exe'
#GradlePath = 'E:/gradle-6.1.1/lib/gradle-launcher-6.1.1.jar'
# NOTE: 需要传给子进程的变量(或者说是除开定义赋值外还有其他赋值操作的非常量)必须走参数传递,multiprocessing.Value 的共享变量的方式有点问题
# 需从 Jenkins 传入的参数
PlatformName = TargetBuildOutputName = UnityPath = ProjRootPath = UinityBuildPipeName = P4Path = UploadPrefix = UploadPath = UploadNotifyWebhook = BuildVersion = BuildType = ''
# Android 专用传入参数
JAVAPath = GradlePath = KeystoreRelPath = ''
# 通过传入参数组装的参数
TargetOutputAbsPath = ResProjPath = AppProjPath = GenBuildProjPath = BuildAppLogPath = BuildResLogPath = OldResInAppPath = TempSaveResPath = ''
# 在所有进程中使用的常量或只在主进程使用的变量
_BuildTimeOutSeconds = 3600
_UinityMethodName = 'GameMenuEditor.BuildAndroid'
_BuildSharedFileFromUnity = 'UnityBuildSharedFile'
_BuildAppDoneMessage = 'AppDone'
_BuildAppErrorMessage = 'AppErr'
_BuildResDoneMessage = 'ResDone'
_BuildResErrorMessage = 'ResErr'
_BuildTargetSuffix = ''
_DefaultBuildTargetPrefix = ''
_BuildAppTime = 0
_BuildResTime = 0
_MoveAssetsTime = 0
_BuildGenProjTime = 0
_AllBuildTime = 0
def InitAllVars(argvs):
# 小写,写法参照 https://docs.unity3d.com/2021.2/Documentation/Manual/EditorCommandLineArguments.html Build Arguments
global PlatformName; PlatformName = argvs[1]
global _IsAndroid, _IsIOS, _IsWin64, _BuildTargetSuffix, _DefaultBuildTargetPrefix
if PlatformName == 'Android':
_IsAndroid = True
_BuildTargetSuffix = '.apk'
_DefaultBuildTargetPrefix = 'launcher'
elif PlatformName == 'iOS':
_IsIOS = True
_BuildTargetSuffix = '.ipa'
_DefaultBuildTargetPrefix = 'Unity-iPhone'
elif PlatformName == 'win64':
_IsWin64 = True
_BuildTargetSuffix = '.zip'
if _IsAndroid:
argCount = 19
elif _IsIOS:
argCount = 23
elif _IsWin64:
argCount = 12
else:
argCount = 0
PrintLogASAP('[ParallelBuild] 所有传入参数分别为:{}\n'.format(len(argvs)))
for arg in argvs:
PrintLogASAP(arg)
PrintLogASAP('[ParallelBuild] 注意核验上述参数...')
if (len(argvs) != argCount):
PrintLogASAP("[ParallelBuild] Error! 传入参数数量错误,或不支持该打包平台!")
return False
# global BuildVersion; BuildVersion = argvs[2]
global TargetBuildOutputName; TargetBuildOutputName = argvs[7] #'test.apk'#argvs[3]12432342
global UnityPath; UnityPath = argvs[2] #'C:/Application/Unity2019.4.26f1c1/Editor/Unity.exe'#
global ProjRootPath; ProjRootPath = argvs[3]#'D:/client_workspace/pandora/pandora_cli_proj' #argvs[5]
global TargetOutputAbsDir; TargetOutputAbsDir = argvs[6]
global TargetOutputAbsPath; TargetOutputAbsPath ="{}/{}".format(TargetOutputAbsDir, TargetBuildOutputName) #'{}/build/{}/{}'.format(ProjRootPath, outputDir, TargetBuildOutputName)
global Version; Version = argvs[9]
global disableLog; disableLog = argvs[10]
# global versionIP; versionIP = argvs[11]
global mode; mode = argvs[11]
global publishChannel; publishChannel = argvs[12]
# global gameIP; gameIP = argvs[14]
# global maintenanceIP; maintenanceIP = argvs[15]
global lang; lang = argvs[13]
global isUsingUWATools; isUsingUWATools = argvs[14]
global config_flag; config_flag = argvs[15]
global config_2_flag; config_2_flag = argvs[16]
global useCustomPackagename; useCustomPackagename = argvs[17]
global isUsingUWAPoco; isUsingUWAPoco = argvs[18]
# global thinkingAnalyticsMode; thinkingAnalyticsMode = argvs[19]
# global P4Path; P4Path = argvs[7]
# global UploadPrefix; UploadPrefix = 'framw' #argvs[8]
# global UploadPath; UploadPath = argvs[9]
# global UploadNotifyWebhook; UploadNotifyWebhook = argvs[10]
if os.path.exists(TargetOutputAbsDir):
try:
shutil.rmtree(TargetOutputAbsDir)
except OSError as e:
PrintLogASAP("Error: %s - %s." % (e.filename, e.strerror))
os.makedirs(TargetOutputAbsDir)
global BuildType;
global Development;
if (argvs[8] == 'true'):
BuildType = 'Debug'
Development = 'true'
else:
BuildType = 'Release'
Development = 'false'
if _IsAndroid:
global KeystoreRelPath; KeystoreRelPath = '../android'
global JAVAPath; JAVAPath = argvs[4]# 'D:/jdk1.8.0_91/jdk1.8.0_91/bin/java.exe' #argvs[13]
global GradlePath; GradlePath = argvs[5]#'C:/Application/Unity2019.4.26f1c1/Editor/Data/PlaybackEngines/AndroidPlayer/Tools/gradle/lib/gradle-launcher-6.1.1.jar' #argvs[14]
outputDir = 'build-{}-output'.format(PlatformName)
global ResProjPath; ResProjPath = '{}/client'.format(ProjRootPath)
PrintLogASAP(ResProjPath)
global AppProjPath; AppProjPath = '{}/anheiTempForPureAPP'.format(ProjRootPath)
global BuildAppLogPath; BuildAppLogPath = '{}/logs/{}BuildApp.log'.format(ProjRootPath, TargetBuildOutputName)
global BuildResLogPath; BuildResLogPath = '{}/logs/{}BuildRes.log'.format(ProjRootPath, TargetBuildOutputName)
global OldResInAppPath; OldResInAppPath = '{}/StreamingAssets'.format(ProjRootPath)
global TempSaveResPath; TempSaveResPath = '{}/StreamingAssets'.format(os.path.dirname(ProjRootPath)) # 迎合 C#脚本
global GenBuildProjPath
if _IsAndroid:
GenBuildProjPath = '{}/build/{}/{}'.format(os.path.dirname(ProjRootPath), PlatformName, 'app') # 迎合 C#脚本
elif _IsIOS:
GenBuildProjPath = '{}/build/{}'.format(AppProjPath, 'XcodeProject')
elif _IsWin64:
GenBuildProjPath = '{}/build/{}'.format(AppProjPath, 'StandaloneWindows64')
return True
def RmErrHandler(func, path, execInfo):
PrintLogASAP('[RmErrHandler] {} has error as following, try to force remove: '.format(path))
traceback.print_exception(*execInfo)
if _IsIOS:
os.chmod(path, stat.S_IWRITE)
func(path)
elif _IsAndroid or _IsWin64:
os.system('del /f /q "{}"'.format(path))
# NOTE: 子进程要用到的变量若是除开定义外还被赋值过的,需要通过传参进来后再使用,常量 _UinityMethodName 这种可以直接用
def ChildProcessBuildAppParallel(connect, argOldResInAppPath, argBuildAppLogPath, argUnityPath, argAppProjPath, argUinityBuildPipeName, argPlatformName, publishChannel, config_flag, isUsingUWATools,config_2_flag,Version,\
lang, useCustomPackagename, isUsingUWAPoco, mode, GenBuildProjPath, Development):
# if os.path.isdir(argOldResInAppPath):
# shutil.rmtree(argOldResInAppPath, onerror=RmErrHandler)
if os.path.exists(argBuildAppLogPath):
os.chmod(argBuildAppLogPath, stat.S_IWRITE)
os.remove(argBuildAppLogPath)
logTail = BuildLogTail(argBuildAppLogPath, '[BuildApp] ')
logTail.start()
# 调用子进程 unity 打包 app,等待完成后子进程退出
command = subprocess.Popen([argUnityPath, '-quit', '-batchmode', '-logfile', argBuildAppLogPath, '-projectPath', argAppProjPath, \
'-buildTarget', argPlatformName, '-executeMethod', "XAsset.Build.AssetBuildScript.RunBuild", \
"graph=Assets/ZXToolkit/AssetGraph/Graph/BuildGraph_Parallel_PureApp.asset", "target={}".format(argPlatformName), \
"publishChannel={}".format(publishChannel), "config_flag={}".format(config_flag),\
"isUsingUWATools={}".format(isUsingUWATools), "isUsingUWAPoco={}".format(isUsingUWAPoco), \
"config_2_flag={}".format(config_2_flag), "version={}".format(Version), \
"lang={}".format(lang), "useCustomPackagename={}".format(useCustomPackagename), \
"parallelBuild={}".format("true"), "mode={}".format(mode), "outpath={}".format(GenBuildProjPath), "debug={}".format(Development)], \
stdout=sys.stdout, stderr=sys.stderr)
# unity_app - quit - batchmode - buildTarget ${target} - projectPath ${pandora_pro_dir} - logFile
# "${log_dir}/${buid_name}.log" - executeMethod
# XAsset.Build.AssetBuildScript.RunBuild
# version =$version
# config_flag =$config_flag
# config_2_flag =$config_2_flag
# isUsingUWAPoco =$isUsingUWAPoco
# outpath =$outpath
# development =$development
# thinkingAnalyticsMode =$thinkingAnalyticsMode
# useCustomPackagename =$useCustomPackagename
# versionIP =$versionIP
# mode =$mode
# publishChannel =$publishChannel
# graph =$build_graph
# target =$target
stdout, stderr = command.communicate()
PrintLogASAP('[BuildAppParallel] ret = {}, out = {}, err = {}.'.format(str(command.returncode), stdout, stderr))
logTail.stop()
if command.returncode == 0:
connect.send(_BuildAppDoneMessage)
else:
connect.send(_BuildAppErrorMessage)
# NOTE: 子进程要用到的变量若是除开定义外还被赋值过的,需要通过传参进来后再使用,常量 _UinityMethodName 这种可以直接用
def ChildProcessBuildResParallel(connect, argBuildResLogPath, argUnityPath, argResProjPath, argUinityBuildPipeName, argTargetOutputAbsPath, argPlatformName, publishChannel, config_flag, isUsingUWATools,config_2_flag,Version,\
lang, useCustomPackagename, isUsingUWAPoco, mode):
if os.path.exists(argBuildResLogPath):
os.chmod(argBuildResLogPath, stat.S_IWRITE)
os.remove(argBuildResLogPath)
# 删除上次打包生成的 apk 和 ipa,在 Res 中删除是因为最终会挪到 Res 工程的目录中
if os.path.exists(argTargetOutputAbsPath):
os.chmod(argTargetOutputAbsPath, stat.S_IWRITE)
os.remove(argTargetOutputAbsPath)
logTail = BuildLogTail(argBuildResLogPath, '[BuildRes] ')
logTail.start()
# 调用子进程 unity 打包 res,等待完成后子进程退出
command = subprocess.Popen([argUnityPath, '-quit', '-batchmode', '-logFile', argBuildResLogPath, '-projectPath', argResProjPath, \
'-buildTarget', argPlatformName, '-executeMethod', "XAsset.Build.AssetBuildScript.RunBuild", \
"graph=Assets/ZXToolkit/AssetGraph/Graph/BuildGraph_Parallel_BuildAssets.asset", "target={}".format(argPlatformName), \
"publishChannel={}".format(publishChannel), "config_flag={}".format(config_flag),\
"isUsingUWATools={}".format(isUsingUWATools), "isUsingUWAPoco={}".format(isUsingUWAPoco), \
"config_2_flag={}".format(config_2_flag), "version={}".format(Version), \
"lang={}".format(lang), "useCustomPackagename={}".format(useCustomPackagename), \
"parallelBuild={}".format("true"), "mode={}".format(mode)],\
stdout=sys.stdout, stderr=sys.stderr)
stdout, stderr = command.communicate()
PrintLogASAP('[BuildResParallel] ret = {}, out = {}, err = {}.'.format(str(command.returncode), stdout, stderr))
logTail.stop()
if command.returncode == 0:
connect.send(_BuildResDoneMessage)
else:
connect.send(_BuildResErrorMessage)
def CheckBuildsAllDone(connectAppRecv, connectResRecv):
totalTime = 0
isAppDone = False
isResDone = False
# 轮询判断,等待两者打包完成
while True:
if (not isAppDone) and connectAppRecv.poll():
msg = connectAppRecv.recv()
if msg == _BuildAppDoneMessage:
isAppDone = True
global _BuildAppTime; _BuildAppTime = totalTime
elif msg == _BuildAppErrorMessage:
PrintLogASAP('[BuildAppParallel] Build APP error time: {}s'.format(totalTime))
return False
if (not isResDone) and connectResRecv.poll():
msg = connectResRecv.recv()
if msg == _BuildResDoneMessage:
isResDone = True
global _BuildResTime; _BuildResTime = totalTime
elif msg == _BuildResErrorMessage:
PrintLogASAP('[BuildResParallel] Build Res error time: {}s'.format(totalTime))
return False
if isAppDone and isResDone:
return True
interval = 2
time.sleep(interval)
totalTime += interval
if (totalTime >= _BuildTimeOutSeconds):
PrintLogASAP('[ParallelBuild] 超时了,请检查!')
return False
def MergeAppAndRes():
T1 = time.perf_counter()
# 安卓需要做获取 Gradle 不压缩后缀名的操作,对应 C# 层 Normal 打包时 AndroidPostBuildProcessor 中的操作
# if _IsAndroid:
# suffiexResSet = set()
# for root, _, files in os.walk(TempSaveResPath):
# for file in files:
# splits = os.path.splitext(file)
# if (len(splits[1]) > 1):
# suffiexResSet.add(splits[1])
#
# gradlePropertiesFilePath = '{}/gradle.properties'.format(GenBuildProjPath)
# if (os.path.exists(gradlePropertiesFilePath)):
# with open(gradlePropertiesFilePath, 'a', encoding='utf-8') as file:
# file.write('NoCompressSuffixes={}'.format(','.join(suffiexResSet)))
# else:
# PrintLogASAP('[MergeAppAndRes] 未找到 {},请检查!'.format(gradlePropertiesFilePath))
# return False
# 移动资源
if _IsAndroid:
assetsPath = '{}/unityLibrary/src/main/assets'.format(GenBuildProjPath)
elif _IsIOS:
assetsPath = '{}/Data/Raw'.format(GenBuildProjPath)
elif _IsWin64:
# 从 unity C# 侧共享文件中取出 ResDirNamePrefix
with open('{}/{}'.format(AppProjPath, _BuildSharedFileFromUnity), 'r', encoding='utf-8') as file:
buildSharedJsonData = json.load(file)
PrintLogASAP('[MergeAppAndRes] Get buildSharedJsonData = {} from {}'.format(buildSharedJsonData, _BuildSharedFileFromUnity))
resDirNamePrefix = buildSharedJsonData['ResDirNamePrefix']
assetsPath = '{}/{}_Data/StreamingAssets'.format(GenBuildProjPath, resDirNamePrefix)
else:
return False
tempPathName = os.path.basename(TempSaveResPath)
destDirPath = assetsPath
PrintLogASAP('[BuildResParallel] Merge src {} -> dst {}'.format(tempPathName, destDirPath))
for root, _, files in os.walk(TempSaveResPath):
# 为了支持多级子目录,不用 copytree 是因为目标目录还有别的文件
if os.path.basename(root) != tempPathName:
relDir = root[root.find(tempPathName) + len(tempPathName) : len(root)]
destDirPath = assetsPath + relDir
else:
destDirPath = assetsPath
if not os.path.isdir(destDirPath):
os.makedirs(destDirPath)
for file in files:
if file.endswith('.meta'):
PrintLogASAP('[MergeAppAndRes] delete file {}'.format(file))
continue;
shutil.copy(os.path.join(root, file), destDirPath)
# shutil.move(os.path.join(root, file), destDirPath)
T2 = time.perf_counter()
returnCode = -1
if _IsAndroid:
PrintLogASAP(JAVAPath)
PrintLogASAP(GradlePath)
PrintLogASAP(BuildType)
if mode == "apk":
command = subprocess.Popen([JAVAPath, '-classpath', GradlePath, 'org.gradle.launcher.GradleMain', 'assemble{}'.format(BuildType), '-x', 'verifyReleaseResources'], cwd=GenBuildProjPath, stdout=sys.stdout, stderr=sys.stderr)
else:
command = subprocess.Popen([JAVAPath, '-classpath', GradlePath, 'org.gradle.launcher.GradleMain', 'bundle{}'.format(BuildType), '-x', 'verifyReleaseResources'], cwd=GenBuildProjPath, stdout=sys.stdout, stderr=sys.stderr)
command.communicate()
returnCode = command.returncode
elif _IsIOS:
# buildXcodePy = 'ipaExportor.py'
# os.system('chmod +x {}/{}'.format(GenBuildProjPath, buildXcodePy))
#
# command = subprocess.Popen(['python3', './{}'.format(buildXcodePy), BuildType],\
# cwd=GenBuildProjPath, stdout=sys.stdout, stderr=sys.stderr)
#
# command.communicate()
returnCode = 0
elif _IsWin64:
shutil.make_archive("{}".format(os.path.splitext(TargetOutputAbsPath)[0]), "zip", GenBuildProjPath)
returnCode = 0
T3 = time.perf_counter()
global _MoveAssetsTime; _MoveAssetsTime = int(T2 - T1)
global _BuildGenProjTime; _BuildGenProjTime = int(T3 - T2)
if returnCode == 0:
if _IsAndroid:
buildVariant = BuildType.lower()
if mode == "apk" :
oldBuildTargetPath = '{}/{}/build/outputs/apk/{}/{}-{}{}'.format(GenBuildProjPath, _DefaultBuildTargetPrefix, buildVariant, _DefaultBuildTargetPrefix, buildVariant, _BuildTargetSuffix)
else:
oldBuildTargetPath = '{}/launcher/build/outputs/bundle/{}/launcher-{}.aab'.format(GenBuildProjPath,
buildVariant,
buildVariant)
PrintLogASAP('[ParallelBuild] TargetOutputAbsPath {}'.format(TargetOutputAbsPath))
os.replace(oldBuildTargetPath, TargetOutputAbsPath)
# elif _IsIOS:
# oldBuildTargetPath = '{}/{}{}'.format(GenBuildProjPath, _DefaultBuildTargetPrefix, _BuildTargetSuffix)
# os.replace(oldBuildTargetPath, TargetOutputAbsPath)
PrintLogASAP('[ParallelBuild] MergeAppAndRes succeed!')
return True
else:
PrintLogASAP('[ParallelBuild] MergeAppAndRes failed!')
return False
def UploadToStorage(beginBuildTime, p4CommitChangelist, p4CommitAuthor):
uploadBuildTargetPrefix = '{}_{}_{}_p4:{}_{}'.format(UploadPrefix, BuildVersion, beginBuildTime, p4CommitChangelist, p4CommitAuthor)
hintDSymbol = False
# 处理 dSymbols 上传
if _IsIOS:
dSymbolSuffix = '.dSYM.zip'
dSymbolZipPath = '{}/{}{}'.format(GenBuildProjPath, _DefaultBuildTargetPrefix, dSymbolSuffix)
# 如果存在的话,理论上只有 Release 会有该文件
if os.path.exists(dSymbolZipPath):
PrintLogASAP("[UploadToStorage] Uploading iOS dSymbol ZIP...")
tools.UploadNexus(UploadPath, dSymbolZipPath, '{}{}'.format(uploadBuildTargetPrefix, dSymbolSuffix))
hintDSymbol = True
uploadBuildTargetName = '{}{}'.format(uploadBuildTargetPrefix, _BuildTargetSuffix)
tools.UploadNexus(UploadPath, TargetOutputAbsPath, uploadBuildTargetName)
tools.NoticeUpload(UploadNotifyWebhook, UploadPath, uploadBuildTargetName, p4CommitChangelist, p4CommitAuthor, _BuildTargetSuffix, hintDSymbol)
if __name__ == '__main__':
T1 = time.perf_counter()
if not InitAllVars(sys.argv):
os._exit(-1)
beginBuildTime = tools.GetTime()
# p4CommitChangelist, p4CommitAuthor = tools.GetP4Info(P4Path)
if not os.path.isdir(AppProjPath):
if _IsAndroid:
prepareParallel = PrepareParallelProj4Android(ResProjPath, AppProjPath, KeystoreRelPath)
elif _IsIOS:
prepareParallel = PrepareParallelProj4IOS(ResProjPath, AppProjPath)
elif _IsWin64:
prepareParallel = PrepareParallelProj4Win64(ResProjPath, AppProjPath)
prepareParallel.Run()
connectApp, connectAppRecv = Pipe()
connectRes, connectResRecv = Pipe()
pApp = Process(target=ChildProcessBuildAppParallel, \
args=(connectApp, OldResInAppPath, BuildAppLogPath, UnityPath, AppProjPath, UinityBuildPipeName, PlatformName, publishChannel, config_flag, isUsingUWATools,config_2_flag,Version,\
lang, useCustomPackagename, isUsingUWAPoco, mode, GenBuildProjPath, Development))
pApp.start()
pRes = Process(target=ChildProcessBuildResParallel, \
args=(connectRes, BuildResLogPath, UnityPath, ResProjPath, UinityBuildPipeName, TargetOutputAbsPath, PlatformName, publishChannel, config_flag, isUsingUWATools,config_2_flag,Version,\
lang, useCustomPackagename, isUsingUWAPoco, mode))
pRes.start()
finalRes = False
if CheckBuildsAllDone(connectAppRecv, connectResRecv):
PrintLogASAP('[ParallelBuild] 开始合并')
finalRes = MergeAppAndRes()
if finalRes:
PrintLogASAP('[ParallelBuild] 合并完成,开始上传')
# UploadToStorage(beginBuildTime, p4CommitChangelist, p4CommitAuthor)
else:
PrintLogASAP('[ParallelBuild] 有并行任务失败了,退出!')
# 关闭所有父子进程
if pApp.is_alive():
appProcess = psutil.Process(pApp.pid)
if appProcess and appProcess.is_running():
for childProc in appProcess.children(recursive=True):
childProc.terminate()
appProcess.terminate()
if pRes.is_alive():
resProcess = psutil.Process(pRes.pid)
if resProcess and resProcess.is_running():
for childProc in resProcess.children(recursive=True):
childProc.terminate()
resProcess.terminate()
connectApp.close()
connectRes.close()
T2 = time.perf_counter()
_AllBuildTime = int(T2 - T1)
PrintLogASAP('[ParallelBuild] All Build time: {}s'.format(_AllBuildTime))
PrintLogASAP('[ParallelBuild] Build APP time: {}s'.format(_BuildAppTime))
PrintLogASAP('[ParallelBuild] Build Res time: {}s'.format(_BuildResTime))
PrintLogASAP('[ParallelBuild] Move Assets time: {}s'.format(_MoveAssetsTime))
PrintLogASAP('[ParallelBuild] Build Gen Project time: {}s'.format(_BuildGenProjTime))
if finalRes == True:
os._exit(0)
else:
os._exit(-1)