from multiprocessing import Process, Pipe import time import os import shutil import subprocess import sys import stat import traceback import psutil import json from PyModules.BuildLogTail import BuildLogTail, PrintLogASAP from PyModules.PrepareParallelProj import PrepareParallelProj4Win64, PrepareParallelProj4Android, PrepareParallelProj4IOS import tools _IsAndroid = False _IsIOS = False _IsWin64 = False # 参数配置举例 #UnityPath = 'C:/Program Files/Unity/Hub/Editor/2018.4.19f1/Editor/Unity.exe' #JAVAPath = 'C:/Program Files/Unity/Hub/Editor/2018.4.19f1/Editor/Data/PlaybackEngines/AndroidPlayer/Tools/OpenJDK/Windows/bin/java.exe' #GradlePath = 'E:/gradle-6.1.1/lib/gradle-launcher-6.1.1.jar' # NOTE: 需要传给子进程的变量（或者说是除开定义赋值外还有其他赋值操作的非常量）必须走参数传递，multiprocessing.Value 的共享变量的方式有点问题 # 需从 Jenkins 传入的参数 PlatformName = TargetBuildOutputName = UnityPath = ProjRootPath = UinityBuildPipeName = P4Path = UploadPrefix = UploadPath = UploadNotifyWebhook = BuildVersion = BuildType = '' # Android 专用传入参数 JAVAPath = GradlePath = KeystoreRelPath = '' # 通过传入参数组装的参数 TargetOutputAbsPath = ResProjPath = AppProjPath = GenBuildProjPath = BuildAppLogPath = BuildResLogPath = OldResInAppPath = TempSaveResPath = '' # 在所有进程中使用的常量或只在主进程使用的变量 _BuildTimeOutSeconds = 3600 _UinityMethodName = 'GameMenuEditor.BuildAndroid' _BuildSharedFileFromUnity = 'UnityBuildSharedFile' _BuildAppDoneMessage = 'AppDone' _BuildAppErrorMessage = 'AppErr' _BuildResDoneMessage = 'ResDone' _BuildResErrorMessage = 'ResErr' _BuildTargetSuffix = '' _DefaultBuildTargetPrefix = '' _BuildAppTime = 0 _BuildResTime = 0 _MoveAssetsTime = 0 _BuildGenProjTime = 0 _AllBuildTime = 0 def InitAllVars(argvs): # 小写，写法参照 https://docs.unity3d.com/2021.2/Documentation/Manual/EditorCommandLineArguments.html Build Arguments global PlatformName; PlatformName = argvs[1] global _IsAndroid, _IsIOS, _IsWin64, _BuildTargetSuffix, _DefaultBuildTargetPrefix if PlatformName == 'Android': _IsAndroid = True _BuildTargetSuffix = '.apk' _DefaultBuildTargetPrefix = 'launcher' elif PlatformName == 'iOS': _IsIOS = True _BuildTargetSuffix = '.ipa' _DefaultBuildTargetPrefix = 'Unity-iPhone' elif PlatformName == 'win64': _IsWin64 = True _BuildTargetSuffix = '.zip' if _IsAndroid: argCount = 19 elif _IsIOS: argCount = 23 elif _IsWin64: argCount = 12 else: argCount = 0 PrintLogASAP('[ParallelBuild] 所有传入参数分别为：{}\n'.format(len(argvs))) for arg in argvs: PrintLogASAP(arg) PrintLogASAP('[ParallelBuild] 注意核验上述参数...') if (len(argvs) != argCount): PrintLogASAP("[ParallelBuild] Error! 传入参数数量错误，或不支持该打包平台!") return False # global BuildVersion; BuildVersion = argvs[2] global TargetBuildOutputName; TargetBuildOutputName = argvs[7] #'test.apk'#argvs[3]12432342 global UnityPath; UnityPath = argvs[2] #'C:/Application/Unity2019.4.26f1c1/Editor/Unity.exe'# global ProjRootPath; ProjRootPath = argvs[3]#'D:/client_workspace/pandora/pandora_cli_proj' #argvs[5] global TargetOutputAbsDir; TargetOutputAbsDir = argvs[6] global TargetOutputAbsPath; TargetOutputAbsPath ="{}/{}".format(TargetOutputAbsDir, TargetBuildOutputName) #'{}/build/{}/{}'.format(ProjRootPath, outputDir, TargetBuildOutputName) global Version; Version = argvs[9] global disableLog; disableLog = argvs[10] # global versionIP; versionIP = argvs[11] global mode; mode = argvs[11] global publishChannel; publishChannel = argvs[12] # global gameIP; gameIP = argvs[14] # global maintenanceIP; maintenanceIP = argvs[15] global lang; lang = argvs[13] global isUsingUWATools; isUsingUWATools = argvs[14] global config_flag; config_flag = argvs[15] global config_2_flag; config_2_flag = argvs[16] global useCustomPackagename; useCustomPackagename = argvs[17] global isUsingUWAPoco; isUsingUWAPoco = argvs[18] # global thinkingAnalyticsMode; thinkingAnalyticsMode = argvs[19] # global P4Path; P4Path = argvs[7] # global UploadPrefix; UploadPrefix = 'framw' #argvs[8] # global UploadPath; UploadPath = argvs[9] # global UploadNotifyWebhook; UploadNotifyWebhook = argvs[10] if os.path.exists(TargetOutputAbsDir): try: shutil.rmtree(TargetOutputAbsDir) except OSError as e: PrintLogASAP("Error: %s - %s." % (e.filename, e.strerror)) os.makedirs(TargetOutputAbsDir) global BuildType; global Development; if (argvs[8] == 'true'): BuildType = 'Debug' Development = 'true' else: BuildType = 'Release' Development = 'false' if _IsAndroid: global KeystoreRelPath; KeystoreRelPath = '../android' global JAVAPath; JAVAPath = argvs[4]# 'D:/jdk1.8.0_91/jdk1.8.0_91/bin/java.exe' #argvs[13] global GradlePath; GradlePath = argvs[5]#'C:/Application/Unity2019.4.26f1c1/Editor/Data/PlaybackEngines/AndroidPlayer/Tools/gradle/lib/gradle-launcher-6.1.1.jar' #argvs[14] outputDir = 'build-{}-output'.format(PlatformName) global ResProjPath; ResProjPath = '{}/client'.format(ProjRootPath) PrintLogASAP(ResProjPath) global AppProjPath; AppProjPath = '{}/anheiTempForPureAPP'.format(ProjRootPath) global BuildAppLogPath; BuildAppLogPath = '{}/logs/{}BuildApp.log'.format(ProjRootPath, TargetBuildOutputName) global BuildResLogPath; BuildResLogPath = '{}/logs/{}BuildRes.log'.format(ProjRootPath, TargetBuildOutputName) global OldResInAppPath; OldResInAppPath = '{}/StreamingAssets'.format(ProjRootPath) global TempSaveResPath; TempSaveResPath = '{}/StreamingAssets'.format(os.path.dirname(ProjRootPath)) # 迎合 C#脚本 global GenBuildProjPath if _IsAndroid: GenBuildProjPath = '{}/build/{}/{}'.format(os.path.dirname(ProjRootPath), PlatformName, 'app') # 迎合 C#脚本 elif _IsIOS: GenBuildProjPath = '{}/build/{}'.format(AppProjPath, 'XcodeProject') elif _IsWin64: GenBuildProjPath = '{}/build/{}'.format(AppProjPath, 'StandaloneWindows64') return True def RmErrHandler(func, path, execInfo): PrintLogASAP('[RmErrHandler] {} has error as following, try to force remove: '.format(path)) traceback.print_exception(*execInfo) if _IsIOS: os.chmod(path, stat.S_IWRITE) func(path) elif _IsAndroid or _IsWin64: os.system('del /f /q "{}"'.format(path)) # NOTE: 子进程要用到的变量若是除开定义外还被赋值过的，需要通过传参进来后再使用，常量 _UinityMethodName 这种可以直接用 def ChildProcessBuildAppParallel(connect, argOldResInAppPath, argBuildAppLogPath, argUnityPath, argAppProjPath, argUinityBuildPipeName, argPlatformName, publishChannel, config_flag, isUsingUWATools,config_2_flag,Version,\ lang, useCustomPackagename, isUsingUWAPoco, mode, GenBuildProjPath, Development): # if os.path.isdir(argOldResInAppPath): # shutil.rmtree(argOldResInAppPath, onerror=RmErrHandler) if os.path.exists(argBuildAppLogPath): os.chmod(argBuildAppLogPath, stat.S_IWRITE) os.remove(argBuildAppLogPath) logTail = BuildLogTail(argBuildAppLogPath, '[BuildApp] ') logTail.start() # 调用子进程 unity 打包 app，等待完成后子进程退出 command = subprocess.Popen([argUnityPath, '-quit', '-batchmode', '-logfile', argBuildAppLogPath, '-projectPath', argAppProjPath, \ '-buildTarget', argPlatformName, '-executeMethod', "XAsset.Build.AssetBuildScript.RunBuild", \ "graph=Assets/ZXToolkit/AssetGraph/Graph/BuildGraph_Parallel_PureApp.asset", "target={}".format(argPlatformName), \ "publishChannel={}".format(publishChannel), "config_flag={}".format(config_flag),\ "isUsingUWATools={}".format(isUsingUWATools), "isUsingUWAPoco={}".format(isUsingUWAPoco), \ "config_2_flag={}".format(config_2_flag), "version={}".format(Version), \ "lang={}".format(lang), "useCustomPackagename={}".format(useCustomPackagename), \ "parallelBuild={}".format("true"), "mode={}".format(mode), "outpath={}".format(GenBuildProjPath), "debug={}".format(Development)], \ stdout=sys.stdout, stderr=sys.stderr) # unity_app - quit - batchmode - buildTarget ${target} - projectPath ${pandora_pro_dir} - logFile # "${log_dir}/${buid_name}.log" - executeMethod # XAsset.Build.AssetBuildScript.RunBuild # version =$version # config_flag =$config_flag # config_2_flag =$config_2_flag # isUsingUWAPoco =$isUsingUWAPoco # outpath =$outpath # development =$development # thinkingAnalyticsMode =$thinkingAnalyticsMode # useCustomPackagename =$useCustomPackagename # versionIP =$versionIP # mode =$mode # publishChannel =$publishChannel # graph =$build_graph # target =$target stdout, stderr = command.communicate() PrintLogASAP('[BuildAppParallel] ret = {}, out = {}, err = {}.'.format(str(command.returncode), stdout, stderr)) logTail.stop() if command.returncode == 0: connect.send(_BuildAppDoneMessage) else: connect.send(_BuildAppErrorMessage) # NOTE: 子进程要用到的变量若是除开定义外还被赋值过的，需要通过传参进来后再使用，常量 _UinityMethodName 这种可以直接用 def ChildProcessBuildResParallel(connect, argBuildResLogPath, argUnityPath, argResProjPath, argUinityBuildPipeName, argTargetOutputAbsPath, argPlatformName, publishChannel, config_flag, isUsingUWATools,config_2_flag,Version,\ lang, useCustomPackagename, isUsingUWAPoco, mode): if os.path.exists(argBuildResLogPath): os.chmod(argBuildResLogPath, stat.S_IWRITE) os.remove(argBuildResLogPath) # 删除上次打包生成的 apk 和 ipa，在 Res 中删除是因为最终会挪到 Res 工程的目录中 if os.path.exists(argTargetOutputAbsPath): os.chmod(argTargetOutputAbsPath, stat.S_IWRITE) os.remove(argTargetOutputAbsPath) logTail = BuildLogTail(argBuildResLogPath, '[BuildRes] ') logTail.start() # 调用子进程 unity 打包 res，等待完成后子进程退出 command = subprocess.Popen([argUnityPath, '-quit', '-batchmode', '-logFile', argBuildResLogPath, '-projectPath', argResProjPath, \ '-buildTarget', argPlatformName, '-executeMethod', "XAsset.Build.AssetBuildScript.RunBuild", \ "graph=Assets/ZXToolkit/AssetGraph/Graph/BuildGraph_Parallel_BuildAssets.asset", "target={}".format(argPlatformName), \ "publishChannel={}".format(publishChannel), "config_flag={}".format(config_flag),\ "isUsingUWATools={}".format(isUsingUWATools), "isUsingUWAPoco={}".format(isUsingUWAPoco), \ "config_2_flag={}".format(config_2_flag), "version={}".format(Version), \ "lang={}".format(lang), "useCustomPackagename={}".format(useCustomPackagename), \ "parallelBuild={}".format("true"), "mode={}".format(mode)],\ stdout=sys.stdout, stderr=sys.stderr) stdout, stderr = command.communicate() PrintLogASAP('[BuildResParallel] ret = {}, out = {}, err = {}.'.format(str(command.returncode), stdout, stderr)) logTail.stop() if command.returncode == 0: connect.send(_BuildResDoneMessage) else: connect.send(_BuildResErrorMessage) def CheckBuildsAllDone(connectAppRecv, connectResRecv): totalTime = 0 isAppDone = False isResDone = False # 轮询判断，等待两者打包完成 while True: if (not isAppDone) and connectAppRecv.poll(): msg = connectAppRecv.recv() if msg == _BuildAppDoneMessage: isAppDone = True global _BuildAppTime; _BuildAppTime = totalTime elif msg == _BuildAppErrorMessage: PrintLogASAP('[BuildAppParallel] Build APP error time: {}s'.format(totalTime)) return False if (not isResDone) and connectResRecv.poll(): msg = connectResRecv.recv() if msg == _BuildResDoneMessage: isResDone = True global _BuildResTime; _BuildResTime = totalTime elif msg == _BuildResErrorMessage: PrintLogASAP('[BuildResParallel] Build Res error time: {}s'.format(totalTime)) return False if isAppDone and isResDone: return True interval = 2 time.sleep(interval) totalTime += interval if (totalTime >= _BuildTimeOutSeconds): PrintLogASAP('[ParallelBuild] 超时了，请检查！') return False def MergeAppAndRes(): T1 = time.perf_counter() # 安卓需要做获取 Gradle 不压缩后缀名的操作，对应 C# 层 Normal 打包时 AndroidPostBuildProcessor 中的操作 # if _IsAndroid: # suffiexResSet = set() # for root, _, files in os.walk(TempSaveResPath): # for file in files: # splits = os.path.splitext(file) # if (len(splits[1]) > 1): # suffiexResSet.add(splits[1]) # # gradlePropertiesFilePath = '{}/gradle.properties'.format(GenBuildProjPath) # if (os.path.exists(gradlePropertiesFilePath)): # with open(gradlePropertiesFilePath, 'a', encoding='utf-8') as file: # file.write('NoCompressSuffixes={}'.format(','.join(suffiexResSet))) # else: # PrintLogASAP('[MergeAppAndRes] 未找到 {}，请检查！'.format(gradlePropertiesFilePath)) # return False # 移动资源 if _IsAndroid: assetsPath = '{}/unityLibrary/src/main/assets'.format(GenBuildProjPath) elif _IsIOS: assetsPath = '{}/Data/Raw'.format(GenBuildProjPath) elif _IsWin64: # 从 unity C# 侧共享文件中取出 ResDirNamePrefix with open('{}/{}'.format(AppProjPath, _BuildSharedFileFromUnity), 'r', encoding='utf-8') as file: buildSharedJsonData = json.load(file) PrintLogASAP('[MergeAppAndRes] Get buildSharedJsonData = {} from {}'.format(buildSharedJsonData, _BuildSharedFileFromUnity)) resDirNamePrefix = buildSharedJsonData['ResDirNamePrefix'] assetsPath = '{}/{}_Data/StreamingAssets'.format(GenBuildProjPath, resDirNamePrefix) else: return False tempPathName = os.path.basename(TempSaveResPath) destDirPath = assetsPath PrintLogASAP('[BuildResParallel] Merge src {} -> dst {}'.format(tempPathName, destDirPath)) for root, _, files in os.walk(TempSaveResPath): # 为了支持多级子目录，不用 copytree 是因为目标目录还有别的文件 if os.path.basename(root) != tempPathName: relDir = root[root.find(tempPathName) + len(tempPathName) : len(root)] destDirPath = assetsPath + relDir else: destDirPath = assetsPath if not os.path.isdir(destDirPath): os.makedirs(destDirPath) for file in files: if file.endswith('.meta'): PrintLogASAP('[MergeAppAndRes] delete file {}'.format(file)) continue; shutil.copy(os.path.join(root, file), destDirPath) # shutil.move(os.path.join(root, file), destDirPath) T2 = time.perf_counter() returnCode = -1 if _IsAndroid: PrintLogASAP(JAVAPath) PrintLogASAP(GradlePath) PrintLogASAP(BuildType) if mode == "apk": command = subprocess.Popen([JAVAPath, '-classpath', GradlePath, 'org.gradle.launcher.GradleMain', 'assemble{}'.format(BuildType), '-x', 'verifyReleaseResources'], cwd=GenBuildProjPath, stdout=sys.stdout, stderr=sys.stderr) else: command = subprocess.Popen([JAVAPath, '-classpath', GradlePath, 'org.gradle.launcher.GradleMain', 'bundle{}'.format(BuildType), '-x', 'verifyReleaseResources'], cwd=GenBuildProjPath, stdout=sys.stdout, stderr=sys.stderr) command.communicate() returnCode = command.returncode elif _IsIOS: # buildXcodePy = 'ipaExportor.py' # os.system('chmod +x {}/{}'.format(GenBuildProjPath, buildXcodePy)) # # command = subprocess.Popen(['python3', './{}'.format(buildXcodePy), BuildType],\ # cwd=GenBuildProjPath, stdout=sys.stdout, stderr=sys.stderr) # # command.communicate() returnCode = 0 elif _IsWin64: shutil.make_archive("{}".format(os.path.splitext(TargetOutputAbsPath)[0]), "zip", GenBuildProjPath) returnCode = 0 T3 = time.perf_counter() global _MoveAssetsTime; _MoveAssetsTime = int(T2 - T1) global _BuildGenProjTime; _BuildGenProjTime = int(T3 - T2) if returnCode == 0: if _IsAndroid: buildVariant = BuildType.lower() if mode == "apk" : oldBuildTargetPath = '{}/{}/build/outputs/apk/{}/{}-{}{}'.format(GenBuildProjPath, _DefaultBuildTargetPrefix, buildVariant, _DefaultBuildTargetPrefix, buildVariant, _BuildTargetSuffix) else: oldBuildTargetPath = '{}/launcher/build/outputs/bundle/{}/launcher-{}.aab'.format(GenBuildProjPath, buildVariant, buildVariant) PrintLogASAP('[ParallelBuild] TargetOutputAbsPath {}'.format(TargetOutputAbsPath)) os.replace(oldBuildTargetPath, TargetOutputAbsPath) # elif _IsIOS: # oldBuildTargetPath = '{}/{}{}'.format(GenBuildProjPath, _DefaultBuildTargetPrefix, _BuildTargetSuffix) # os.replace(oldBuildTargetPath, TargetOutputAbsPath) PrintLogASAP('[ParallelBuild] MergeAppAndRes succeed!') return True else: PrintLogASAP('[ParallelBuild] MergeAppAndRes failed!') return False def UploadToStorage(beginBuildTime, p4CommitChangelist, p4CommitAuthor): uploadBuildTargetPrefix = '{}_{}_{}_p4:{}_{}'.format(UploadPrefix, BuildVersion, beginBuildTime, p4CommitChangelist, p4CommitAuthor) hintDSymbol = False # 处理 dSymbols 上传 if _IsIOS: dSymbolSuffix = '.dSYM.zip' dSymbolZipPath = '{}/{}{}'.format(GenBuildProjPath, _DefaultBuildTargetPrefix, dSymbolSuffix) # 如果存在的话，理论上只有 Release 会有该文件 if os.path.exists(dSymbolZipPath): PrintLogASAP("[UploadToStorage] Uploading iOS dSymbol ZIP...") tools.UploadNexus(UploadPath, dSymbolZipPath, '{}{}'.format(uploadBuildTargetPrefix, dSymbolSuffix)) hintDSymbol = True uploadBuildTargetName = '{}{}'.format(uploadBuildTargetPrefix, _BuildTargetSuffix) tools.UploadNexus(UploadPath, TargetOutputAbsPath, uploadBuildTargetName) tools.NoticeUpload(UploadNotifyWebhook, UploadPath, uploadBuildTargetName, p4CommitChangelist, p4CommitAuthor, _BuildTargetSuffix, hintDSymbol) if __name__ == '__main__': T1 = time.perf_counter() if not InitAllVars(sys.argv): os._exit(-1) beginBuildTime = tools.GetTime() # p4CommitChangelist, p4CommitAuthor = tools.GetP4Info(P4Path) if not os.path.isdir(AppProjPath): if _IsAndroid: prepareParallel = PrepareParallelProj4Android(ResProjPath, AppProjPath, KeystoreRelPath) elif _IsIOS: prepareParallel = PrepareParallelProj4IOS(ResProjPath, AppProjPath) elif _IsWin64: prepareParallel = PrepareParallelProj4Win64(ResProjPath, AppProjPath) prepareParallel.Run() connectApp, connectAppRecv = Pipe() connectRes, connectResRecv = Pipe() pApp = Process(target=ChildProcessBuildAppParallel, \ args=(connectApp, OldResInAppPath, BuildAppLogPath, UnityPath, AppProjPath, UinityBuildPipeName, PlatformName, publishChannel, config_flag, isUsingUWATools,config_2_flag,Version,\ lang, useCustomPackagename, isUsingUWAPoco, mode, GenBuildProjPath, Development)) pApp.start() pRes = Process(target=ChildProcessBuildResParallel, \ args=(connectRes, BuildResLogPath, UnityPath, ResProjPath, UinityBuildPipeName, TargetOutputAbsPath, PlatformName, publishChannel, config_flag, isUsingUWATools,config_2_flag,Version,\ lang, useCustomPackagename, isUsingUWAPoco, mode)) pRes.start() finalRes = False if CheckBuildsAllDone(connectAppRecv, connectResRecv): PrintLogASAP('[ParallelBuild] 开始合并') finalRes = MergeAppAndRes() if finalRes: PrintLogASAP('[ParallelBuild] 合并完成，开始上传') # UploadToStorage(beginBuildTime, p4CommitChangelist, p4CommitAuthor) else: PrintLogASAP('[ParallelBuild] 有并行任务失败了，退出！') # 关闭所有父子进程 if pApp.is_alive(): appProcess = psutil.Process(pApp.pid) if appProcess and appProcess.is_running(): for childProc in appProcess.children(recursive=True): childProc.terminate() appProcess.terminate() if pRes.is_alive(): resProcess = psutil.Process(pRes.pid) if resProcess and resProcess.is_running(): for childProc in resProcess.children(recursive=True): childProc.terminate() resProcess.terminate() connectApp.close() connectRes.close() T2 = time.perf_counter() _AllBuildTime = int(T2 - T1) PrintLogASAP('[ParallelBuild] All Build time: {}s'.format(_AllBuildTime)) PrintLogASAP('[ParallelBuild] Build APP time: {}s'.format(_BuildAppTime)) PrintLogASAP('[ParallelBuild] Build Res time: {}s'.format(_BuildResTime)) PrintLogASAP('[ParallelBuild] Move Assets time: {}s'.format(_MoveAssetsTime)) PrintLogASAP('[ParallelBuild] Build Gen Project time: {}s'.format(_BuildGenProjTime)) if finalRes == True: os._exit(0) else: os._exit(-1)