This commit is contained in:
admin_d committed
1 parent
15a968a2f7
commit
b52921e171
38 files changed
+564
-2956
No files matched your search
+137
-231
@@ -1,25 +1,19 @@
|
||||
"""One-command Android-only compile, full native link, APK and evidence pipeline."""
|
||||
from __future__ import annotations
|
||||
import argparse, json, os, re, shutil, subprocess, sys, uuid, zipfile
|
||||
import argparse, json, os, re, shutil, subprocess, uuid, zipfile
|
||||
from pathlib import Path
|
||||
from common import ROOT, COMMIT, CPUS, ProbeError, stamp, sha256, write_json, run_command, first_errors, classify
|
||||
from common import (ROOT, COMMIT, CPUS, ProbeError, stamp, sha256, write_json, run_command,
|
||||
first_errors, CURRENT_OUTPUT_DIR, CURRENT_BUILD_METADATA,
|
||||
CURRENT_PACKAGE_METADATA, CURRENT_APK, CURRENT_BUILD_SCHEMA,
|
||||
CURRENT_PACKAGE_SCHEMA)
|
||||
from source_fetch import fetch, verify
|
||||
from source_audit import inventory
|
||||
from source_resolution import load_source_contract, validate_source_contract
|
||||
from generate import make_target, asset_inventory
|
||||
from source_resolution import load_source_contract, validate_source_contract, build_source_resolution
|
||||
from generate import materialize_application_project
|
||||
from artifacts import inspect_elf, inspect_apk
|
||||
from resource_staging import stage_runtime_resources
|
||||
|
||||
IMAGE_TAG='kambi/castle-engine-cloud-builds-tools:cge-unstable'
|
||||
PROFILES=json.loads((ROOT/'config/profiles.json').read_text())
|
||||
|
||||
def selected_profiles(text:str)->list[dict]:
|
||||
if text=='all':return PROFILES
|
||||
names=[s.strip() for s in text.split(',') if s.strip()]
|
||||
available={p['id'] for p in PROFILES}
|
||||
if not names or set(names)-available:raise ProbeError('Unknown/empty profiles: '+text)
|
||||
names=set(names)|{'platform'} # Always establish the same-toolchain platform baseline.
|
||||
return [p for p in PROFILES if p['id'] in names]
|
||||
|
||||
def cge_command(cpu:str,action:str)->list[str]:
|
||||
if cpu not in CPUS:raise ProbeError('Unsupported Android CPU')
|
||||
if action not in ('compile','package'):raise ProbeError('Not an Android build action')
|
||||
@@ -27,73 +21,6 @@ def cge_command(cpu:str,action:str)->list[str]:
|
||||
if action=='package':cmd+=['--assume-compiled','--package-format=android-apk']
|
||||
return cmd+[action]
|
||||
|
||||
def blocked_row(profile:dict,reason:str)->dict:
|
||||
return {'id':profile['id'],'required':profile['required'],'purpose':profile['purpose'],
|
||||
'compile_link':{'status':'BLOCKED','reason':reason},'package':{'status':'NOT_RUN'},
|
||||
'runtime':{'status':'NOT_RUN'},'errors':[]}
|
||||
|
||||
def gate(rows:list[dict])->str:
|
||||
by_id={r['id']:r for r in rows}
|
||||
required=[p for p in PROFILES if p['required']]
|
||||
if any(r['compile_link']['status']=='FAIL' or r['package']['status']=='FAIL' for r in rows if r['required']):
|
||||
return 'NOT_BUILDABLE_AS_IS'
|
||||
if any(r['compile_link']['status']=='BLOCKED' for r in rows if r['required']):return 'BLOCKED'
|
||||
if any(p['id'] not in by_id for p in required):return 'PARTIAL_SELECTION'
|
||||
for p in required:
|
||||
r=by_id[p['id']]
|
||||
if r['compile_link']['status']!='PASS':return 'BLOCKED'
|
||||
if p['package'] and r['package']['status']!='PASS':return 'BLOCKED'
|
||||
return 'ANDROID_BUILD_GATE_PASS'
|
||||
|
||||
def render_summary(result:dict,out:Path)->None:
|
||||
title='# KaM r16020 — Android architecture probe\n\n'
|
||||
title+=f"Build ID: `{result['run_id']}` · CPU: `{result['cpu']}` · Gate: **{result['gate']}**\n\n"
|
||||
title+='Нет desktop target. Успех означает сборку выбранных Android-корней, не игровую совместимость.\n\n'
|
||||
title+='| Проба | Compile + native link | APK | Ошибка / следующий шаг |\n|---|---|---|---|\n'
|
||||
for r in result['profiles']:
|
||||
error=(r.get('errors') or [r['compile_link'].get('reason','')])[0]
|
||||
error=error.replace('|','/').replace('\n',' ')[:260]
|
||||
title+=f"| {r['id']} | {r['compile_link']['status']} | {r['package']['status']} | {error} |\n"
|
||||
title+='\n## Что установлено\n\n'
|
||||
title+='`PASS` compile_link = FPC завершился успешно, найден свежий ELF нужного ABI, проверены корневые PPU. Линковка не отключена; требуются определённые символы. '
|
||||
title+='`PASS` APK = упаковка завершилась, проверены ZIP/DEX, правильный ABI, совпадение native library и DT_NEEDED. '
|
||||
title+='Подпись проверяется отдельно, если доступен apksigner. `runtime=NOT_RUN` до реального запуска на Android.\n\n'
|
||||
title+='`NOT_BUILDABLE_AS_IS` — найдено препятствие в текущем коде/окружении, а не доказательство, что core надо переписать. '
|
||||
title+='Ошибки legacy_render/legacy_audio диагностические и не входят в обязательный gate, но их транзитивное попадание в core всё равно видно.\n\n'
|
||||
title+='## Как принимать решение\n\n'
|
||||
title+='| Свидетельство | Разумное решение |\n|---|---|\n'
|
||||
title+='| Корень собирается с оригинальными исходниками | Кандидат на reuse; поведение ещё проверяется отдельно |\n'
|
||||
title+='| Активная зависимость Windows/LCL/desktop GL | Исследовать минимальное отделение платформенной части |\n'
|
||||
title+='| Отсутствует source unit / package | Проверить комплектность исходников, условные defines и пакет для Android |\n'
|
||||
title+='| Undefined reference / отсутствует native library | Сборка/подключение Android-библиотеки либо адаптер Castle |\n'
|
||||
title+='| Один и тот же блокер у многих модулей | Исправить общий корень и повторить пробы; не считать это множеством отдельных переписываний |\n'
|
||||
title+='| Срабатывает exception при запуске | Снять logcat; проверить unit initialization/lifecycle/доступ к данным |\n'
|
||||
title+='\nНи один автоматический результат здесь не выносит решение «написать подсистему с нуля».\n'
|
||||
(out/'SUMMARY.md').write_text(title,encoding='utf-8')
|
||||
|
||||
def finalize(result:dict,out:Path)->int:
|
||||
result['gate']=gate(result['profiles'])
|
||||
write_json(out/'result.json',result);render_summary(result,out)
|
||||
write_json(ROOT/'reports/LATEST.json',{'run_id':result['run_id'],'directory':str(out.relative_to(ROOT)),
|
||||
'gate':result['gate'],'summary':str((out/'SUMMARY.md').relative_to(ROOT))})
|
||||
shutil.copyfile(out/'SUMMARY.md',ROOT/'reports/LATEST-SUMMARY.md')
|
||||
# Evidence intentionally excludes APK/.so, user game assets, caches and source trees.
|
||||
with zipfile.ZipFile(out/'evidence.zip','w',zipfile.ZIP_DEFLATED) as z:
|
||||
for p in sorted(out.rglob('*')):
|
||||
if p.is_file() and p.suffix in ('.json','.log','.md','.txt','.xml','.pas'):
|
||||
z.write(p,p.relative_to(out))
|
||||
print('\n'+result['gate'])
|
||||
print('Report: '+str(out/'SUMMARY.md'))
|
||||
print('Evidence (no game assets): '+str(out/'evidence.zip'))
|
||||
return 0 if result['gate']=='ANDROID_BUILD_GATE_PASS' else 1 if result['gate']=='NOT_BUILDABLE_AS_IS' else 2
|
||||
|
||||
def new_result(cpu:str,profiles:list[dict],reason:str='Not started')->tuple[dict,Path]:
|
||||
rid=stamp()+'-'+uuid.uuid4().hex[:8]
|
||||
out=ROOT/'reports/runs'/rid;out.mkdir(parents=True)
|
||||
return ({'schema':'kam-android-build/1','run_id':rid,'cpu':cpu,'commit':COMMIT,
|
||||
'desktop_target':False,'shims':False,'runtime':'NOT_RUN',
|
||||
'profiles':[blocked_row(p,reason) for p in profiles]},out)
|
||||
|
||||
def tool_versions(out:Path)->dict:
|
||||
commands={'castle':['castle-engine','--version'],'fpc':['fpc','-iV'],'java':['java','-version'],
|
||||
'adb':['adb','version']}
|
||||
@@ -122,175 +49,154 @@ def signature_tool()->str|None:
|
||||
if os.environ.get(key):paths+=list(Path(os.environ[key]).glob('build-tools/*/apksigner'))
|
||||
return str(sorted(paths)[-1]) if paths else None
|
||||
|
||||
def build_local(a:argparse.Namespace,profiles:list[dict])->int:
|
||||
result,out=new_result(a.cpu,profiles)
|
||||
result['toolchain']=tool_versions(out)
|
||||
if not shutil.which('castle-engine'):
|
||||
for r in result['profiles']:r['compile_link']['reason']='castle-engine not installed; no Android compilation attempted'
|
||||
return finalize(result,out)
|
||||
source=ROOT/'vendor/kam_remake';audit=None;validated=None;assets=[]
|
||||
need_source=any(p['units'] for p in profiles)
|
||||
try:
|
||||
contract=load_source_contract()
|
||||
assets=asset_inventory(Path(a.assets),a.asset_path) if a.assets else []
|
||||
result['asset_count']=len(assets)
|
||||
if need_source:
|
||||
if (source/'SOURCE_RECEIPT.json').is_file():receipt=verify(source)
|
||||
elif a.no_fetch:raise ProbeError('Pinned source tree absent and --no-fetch requested')
|
||||
else:receipt=fetch(source)
|
||||
write_json(out/'source-receipt.json',receipt)
|
||||
audit=inventory(source);write_json(out/'source-inventory.json',audit)
|
||||
validated=validate_source_contract(contract,source,receipt,audit)
|
||||
except (ProbeError,OSError,ValueError,KeyError) as e:
|
||||
result['source_or_inputs_error']=str(e)
|
||||
return finalize(result,out)
|
||||
work=ROOT/'build'/result['run_id'];work.mkdir(parents=True)
|
||||
dist=ROOT/'dist'/result['run_id'];dist.mkdir(parents=True)
|
||||
package_index=[]
|
||||
for idx,profile in enumerate(profiles):
|
||||
row=result['profiles'][idx];ident=profile['id'];project=work/ident
|
||||
print(f"[{idx+1}/{len(profiles)}] Android {a.cpu}: {ident}",flush=True)
|
||||
try:
|
||||
if profile['units']:
|
||||
missing=[u for u in profile['units'] if u.casefold() not in audit['units']]
|
||||
if missing:raise ProbeError('Missing required original source units: '+', '.join(missing))
|
||||
# Resources only enter APKs when explicitly requested. The platform baseline remains independent.
|
||||
pack=profile['package'] or a.package_all
|
||||
use_assets=bool(a.assets) and pack and ident=='platform'
|
||||
description=make_target(project,profile,a.cpu,result['run_id'],source if profile['units'] else None,
|
||||
validated if profile['units'] else None,
|
||||
Path(a.assets) if use_assets else None,assets if use_assets else [])
|
||||
row['target']=description
|
||||
evidence=out/ident;evidence.mkdir()
|
||||
for file in ('CastleEngineManifest.xml','target.json'):
|
||||
shutil.copyfile(project/file,evidence/file)
|
||||
shutil.copyfile(project/'code/ProbeImports.pas',evidence/'ProbeImports.pas')
|
||||
row['compile_link']=run_command(cge_command(a.cpu,'compile'),project,evidence/'compile-link.log',a.timeout)
|
||||
text=(evidence/'compile-link.log').read_text(errors='replace')
|
||||
if row['compile_link']['status']!='PASS':
|
||||
row['errors']=first_errors(text) or [text[-1500:]];row['triage']=classify(text);continue
|
||||
name='lib'+description['name']+'.so'
|
||||
outputs=list(project.rglob('lib'+description['name']+'*.so'))
|
||||
if not outputs:raise ProbeError('Compiler exited 0 but no expected native library matching: '+name)
|
||||
hashes={sha256(p) for p in outputs}
|
||||
if len(hashes)!=1:raise ProbeError('Ambiguous different native outputs')
|
||||
library=outputs[0];name=library.name;elf=inspect_elf(library.read_bytes(),a.cpu)
|
||||
row['native_library']={'file':str(library.relative_to(ROOT)),'sha256':sha256(library),'elf':elf}
|
||||
ppu_files={p.stem.casefold():p for p in project.rglob('*.ppu')}
|
||||
evidence_units=[]
|
||||
for unit in description['units']:
|
||||
p=ppu_files.get(unit.casefold())
|
||||
if p is None or p.read_bytes()[:3]!=b'PPU':
|
||||
raise ProbeError('No fresh Pascal unit artifact found for '+unit)
|
||||
evidence_units.append({'unit':unit,'ppu':str(p.relative_to(ROOT)),'sha256':sha256(p),
|
||||
'source_candidates':audit['units'].get(unit.casefold(),[])})
|
||||
row['compiled_roots']=evidence_units
|
||||
if not pack:continue
|
||||
row['package']=run_command(cge_command(a.cpu,'package'),project,evidence/'package.log',a.timeout)
|
||||
if row['package']['status']!='PASS':
|
||||
row['errors']+=first_errors((evidence/'package.log').read_text(errors='replace'));continue
|
||||
candidates=list(project.rglob('*.apk'))
|
||||
if not candidates:raise ProbeError('Packaging exited 0 but no new APK')
|
||||
apk_hashes={sha256(p) for p in candidates}
|
||||
if len(apk_hashes)!=1:raise ProbeError('Multiple different APK outputs; refusing arbitrary selection')
|
||||
apk=candidates[0]
|
||||
apk_info=inspect_apk(apk,a.cpu,name,row['native_library']['sha256'],library.read_bytes())
|
||||
signer=signature_tool()
|
||||
if signer:
|
||||
sign=run_command([signer,'verify','--verbose',str(apk)],project,evidence/'signature.log',120)
|
||||
row['signature']=sign
|
||||
if sign['status']!='PASS':row['package']['status']='FAIL';continue
|
||||
else:row['signature']={'status':'NOT_RUN','reason':'apksigner not found'}
|
||||
final=dist/(description['name']+'-'+a.cpu+'-debug.apk');shutil.copy2(apk,final)
|
||||
row['apk']={'file':str(final.relative_to(ROOT)),**apk_info}
|
||||
package_index.append({'profile':ident,'package':description['package'],'file':str(final.relative_to(ROOT)),
|
||||
'sha256':sha256(final),'cpu':a.cpu,'build_id':result['run_id'],
|
||||
'native_library_path_in_apk':apk_info['main_library_path']})
|
||||
except (ProbeError,OSError,ValueError,KeyError,zipfile.BadZipFile) as e:
|
||||
row['errors'].append(str(e))
|
||||
# A supposedly successful compiler/packager without valid output is a real test failure.
|
||||
if row['package']['status']=='PASS':row['package']['status']='FAIL'
|
||||
elif row['compile_link']['status']=='PASS':row['compile_link']['status']='FAIL'
|
||||
else:row['compile_link']={'status':'BLOCKED','reason':str(e)}
|
||||
finally:
|
||||
write_json(out/'result-in-progress.json',result)
|
||||
if need_source:
|
||||
try:verify(source)
|
||||
except ProbeError as e:
|
||||
result['source_modified_during_build']=str(e)
|
||||
for r in result['profiles']:
|
||||
if r['id']!='platform':r['compile_link']['status']='FAIL';r['errors'].append(str(e))
|
||||
write_json(dist/'apks.json',{'build_id':result['run_id'],'packages':package_index})
|
||||
write_json(ROOT/'dist/LATEST.json',{'build_id':result['run_id'],'directory':str(dist.relative_to(ROOT)),
|
||||
'packages':package_index})
|
||||
return finalize(result,out)
|
||||
def _application(descriptor:dict|None)->dict|None:
|
||||
if descriptor is None:return None
|
||||
return {key:descriptor[key] for key in ('name','caption','package','version','version_code','game_unit')}
|
||||
|
||||
def docker_build(a:argparse.Namespace,profiles:list[dict])->int:
|
||||
result,out=new_result(a.cpu,profiles,'Docker bootstrap not completed')
|
||||
if not shutil.which('docker'):
|
||||
for r in result['profiles']:r['compile_link']['reason']='Docker not installed. Use --local only with a configured CGE/FPC Android toolchain.'
|
||||
return finalize(result,out)
|
||||
def _record(build_id:str, cpu:str, status:str, application:dict|None=None,
|
||||
resource_count:int|None=None, compile:dict|None=None, package:dict|None=None,
|
||||
package_metadata:str|None=None, error:str|None=None)->dict:
|
||||
return {'schema':CURRENT_BUILD_SCHEMA,'build_id':build_id,'status':status,'cpu':cpu,'abi':CPUS[cpu][0],
|
||||
'application':application,'upstream_commit':COMMIT,'resource_count':resource_count,
|
||||
'compile':compile or {'status':'NOT_RUN','exit_code':None},
|
||||
'package':package or {'status':'NOT_RUN','exit_code':None},
|
||||
'package_metadata':package_metadata,'error':error}
|
||||
|
||||
def _finish(record:dict)->int:
|
||||
if record['status']!='PASS':
|
||||
CURRENT_OUTPUT_DIR.mkdir(parents=True,exist_ok=True)
|
||||
for path in (CURRENT_PACKAGE_METADATA,CURRENT_APK):
|
||||
path.unlink(missing_ok=True)
|
||||
record['package_metadata']=None
|
||||
write_json(CURRENT_BUILD_METADATA,record)
|
||||
return {'PASS':0,'FAIL':1,'BLOCKED':2}[record['status']]
|
||||
|
||||
def _valid_docker_handoff(record:object,build_id:str,code:int)->bool:
|
||||
fields={'schema','build_id','status','cpu','abi','application','upstream_commit','resource_count','compile','package','package_metadata','error'}
|
||||
statuses={'PASS':0,'FAIL':1,'BLOCKED':2};stage_statuses=set(statuses)|{'NOT_RUN'}
|
||||
if not isinstance(record,dict) or set(record)!=fields or record.get('schema')!=CURRENT_BUILD_SCHEMA or record.get('build_id')!=build_id or statuses.get(record.get('status'))!=code:return False
|
||||
cpu=record.get('cpu')
|
||||
if cpu not in CPUS or record.get('abi')!=CPUS[cpu][0] or record.get('upstream_commit')!=COMMIT:return False
|
||||
if record['resource_count'] is not None and (type(record['resource_count']) is not int or record['resource_count']<0):return False
|
||||
app=record['application']
|
||||
if app is not None and (not isinstance(app,dict) or set(app)!={'name','caption','package','version','version_code','game_unit'} or any(not isinstance(app[key],str) or not app[key] for key in ('name','caption','package','version','game_unit')) or type(app['version_code']) is not int):return False
|
||||
for stage in ('compile','package'):
|
||||
value=record[stage]
|
||||
if not isinstance(value,dict) or set(value)!={'status','exit_code'} or value.get('status') not in stage_statuses:return False
|
||||
if (value['status']=='NOT_RUN') != (value['exit_code'] is None) or value['exit_code'] is not None and type(value['exit_code']) is not int:return False
|
||||
if record['status']=='PASS':
|
||||
return app is not None and record['resource_count'] is not None and record['compile']['status']==record['package']['status']=='PASS' and record['package_metadata']=='out/current/package.json' and record['error'] is None
|
||||
return record['package_metadata'] is None and isinstance(record['error'],str) and bool(record['error'])
|
||||
|
||||
def _error(log:Path)->str:
|
||||
text=log.read_text(errors='replace') if log.exists() else ''
|
||||
return (first_errors(text) or [text[-1500:] or 'Build command failed'])[0]
|
||||
|
||||
def _prepare(build_id:str, diagnostics:bool=True)->tuple[Path,Path]:
|
||||
work=ROOT/'build'/build_id; logs=work/'logs'
|
||||
logs.mkdir(parents=True,exist_ok=True)
|
||||
if diagnostics:tool_versions(logs)
|
||||
return work,logs
|
||||
|
||||
def build_local(a:argparse.Namespace,build_id:str)->int:
|
||||
work,logs=_prepare(build_id, not bool(os.environ.get('KAM_BUILD_ID')))
|
||||
app=None; resources=None; compile={'status':'NOT_RUN','exit_code':None}; package={'status':'NOT_RUN','exit_code':None}
|
||||
try:
|
||||
vendor_root=ROOT/'vendor/kam_remake'
|
||||
receipt=verify(vendor_root) if (vendor_root/'SOURCE_RECEIPT.json').is_file() else (fetch(vendor_root) if not a.no_fetch else (_ for _ in ()).throw(ProbeError('Pinned source tree absent and --no-fetch requested')))
|
||||
audited=inventory(vendor_root)
|
||||
validated=validate_source_contract(load_source_contract(),vendor_root,receipt,audited)
|
||||
resolution=build_source_resolution(validated,vendor_root,ROOT/'code')
|
||||
descriptor=materialize_application_project(work/'application',resolution);app=_application(descriptor)
|
||||
resources=stage_runtime_resources(descriptor['data_root'])
|
||||
except (ProbeError,OSError,ValueError,KeyError) as error:
|
||||
return _finish(_record(build_id,a.cpu,'BLOCKED',app,None,compile,package,error=str(error)))
|
||||
if not shutil.which('castle-engine'):
|
||||
return _finish(_record(build_id,a.cpu,'BLOCKED',app,len(resources),compile,package,error='castle-engine not installed'))
|
||||
project=descriptor['project_dir']; compile_result=run_command(cge_command(a.cpu,'compile'),project,logs/'compile.log',a.timeout)
|
||||
compile={'status':compile_result['status'],'exit_code':compile_result['exit_code']}
|
||||
if compile['status']!='PASS':
|
||||
return _finish(_record(build_id,a.cpu,compile['status'],app,len(resources),compile,package,error=_error(logs/'compile.log')))
|
||||
try:
|
||||
candidates=list(project.rglob(f"lib{descriptor['name']}_android_{a.cpu}.so"))
|
||||
if len(candidates)!=1:raise ProbeError('Compiler exited 0 but expected exactly one native library')
|
||||
library=candidates[0]; inspect_elf(library.read_bytes(),a.cpu)
|
||||
except (ProbeError,OSError,ValueError) as error:
|
||||
compile['status']='FAIL'
|
||||
return _finish(_record(build_id,a.cpu,'FAIL',app,len(resources),compile,package,error=str(error)))
|
||||
package_result=run_command(cge_command(a.cpu,'package'),project,logs/'package.log',a.timeout)
|
||||
package={'status':package_result['status'],'exit_code':package_result['exit_code']}
|
||||
if package['status']!='PASS':
|
||||
return _finish(_record(build_id,a.cpu,package['status'],app,len(resources),compile,package,error=_error(logs/'package.log')))
|
||||
try:
|
||||
candidates=list(project.rglob('*.apk'))
|
||||
if not candidates:raise ProbeError('Packaging exited 0 but no APK')
|
||||
if len({sha256(path) for path in candidates})!=1:raise ProbeError('Multiple different APK outputs')
|
||||
apk=candidates[0]; apk_info=inspect_apk(apk,a.cpu,library.name,sha256(library),library.read_bytes())
|
||||
signer=signature_tool()
|
||||
if signer:
|
||||
signed=run_command([signer,'verify','--verbose',str(apk)],project,logs/'signature.log',a.timeout)
|
||||
if signed['status']!='PASS':raise ProbeError(_error(logs/'signature.log'))
|
||||
except (ProbeError,OSError,ValueError,KeyError,zipfile.BadZipFile) as error:
|
||||
package['status']='FAIL'
|
||||
return _finish(_record(build_id,a.cpu,'FAIL',app,len(resources),compile,package,error=str(error)))
|
||||
try:
|
||||
CURRENT_OUTPUT_DIR.mkdir(parents=True,exist_ok=True)
|
||||
temporary=CURRENT_APK.with_name(CURRENT_APK.name+'.tmp')
|
||||
shutil.copyfile(apk,temporary);os.replace(temporary,CURRENT_APK)
|
||||
package_data={'schema':CURRENT_PACKAGE_SCHEMA,'build_id':build_id,
|
||||
'application':{key:app[key] for key in ('name','package','version','version_code')},
|
||||
'cpu':a.cpu,'abi':CPUS[a.cpu][0],
|
||||
'apk':{'file':'out/current/kam-mobile.apk','sha256':sha256(CURRENT_APK),'size':CURRENT_APK.stat().st_size,
|
||||
'native_library_path_in_apk':apk_info['main_library_path']}}
|
||||
write_json(CURRENT_PACKAGE_METADATA,package_data)
|
||||
return _finish(_record(build_id,a.cpu,'PASS',app,len(resources),compile,package,
|
||||
'out/current/package.json',None))
|
||||
except (OSError,ValueError,KeyError) as error:
|
||||
package['status']='FAIL'
|
||||
return _finish(_record(build_id,a.cpu,'FAIL',app,len(resources),compile,package,error=str(error)))
|
||||
|
||||
def docker_build(a:argparse.Namespace,build_id:str)->int:
|
||||
compile={'status':'NOT_RUN','exit_code':None};package={'status':'NOT_RUN','exit_code':None}
|
||||
try:
|
||||
work,logs=_prepare(build_id)
|
||||
if not shutil.which('docker'):raise ProbeError('Docker not installed')
|
||||
lock=ROOT/'config/toolchain.lock.json'
|
||||
if lock.is_file():
|
||||
image=json.loads(lock.read_text())['image']
|
||||
if not re.fullmatch(r'[\w./-]+@sha256:[a-f0-9]{64}',image):raise ProbeError('Invalid toolchain digest lock')
|
||||
p=subprocess.run(['docker','image','inspect',image],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)
|
||||
if p.returncode:
|
||||
r=run_command(['docker','pull','--platform=linux/amd64',image],ROOT,out/'docker-pull.log',a.timeout)
|
||||
if r['status']!='PASS':raise ProbeError('Cannot obtain locked Docker image; see docker-pull.log')
|
||||
if subprocess.run(['docker','image','inspect',image],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL).returncode:
|
||||
if run_command(['docker','pull','--platform=linux/amd64',image],ROOT,logs/'docker-pull.log',a.timeout)['status']!='PASS':raise ProbeError('Cannot obtain locked Docker image')
|
||||
else:
|
||||
r=run_command(['docker','pull','--platform=linux/amd64',IMAGE_TAG],ROOT,out/'docker-pull.log',a.timeout)
|
||||
if r['status']!='PASS':raise ProbeError('Docker image pull failed; see docker-pull.log')
|
||||
raw=subprocess.check_output(['docker','image','inspect',IMAGE_TAG],text=True)
|
||||
info=json.loads(raw)[0];digests=info.get('RepoDigests',[])
|
||||
image=next((d for d in digests if 'castle-engine-cloud-builds-tools@sha256:' in d),None)
|
||||
if run_command(['docker','pull','--platform=linux/amd64',IMAGE_TAG],ROOT,logs/'docker-pull.log',a.timeout)['status']!='PASS':raise ProbeError('Docker image pull failed')
|
||||
info=json.loads(subprocess.check_output(['docker','image','inspect',IMAGE_TAG],text=True))[0]
|
||||
image=next((value for value in info.get('RepoDigests',[]) if 'castle-engine-cloud-builds-tools@sha256:' in value),None)
|
||||
if image is None:raise ProbeError('Image has no immutable RepoDigest')
|
||||
write_json(lock,{'image':image,'image_id':info['Id'],'resolved_from':IMAGE_TAG,'locked_at':stamp(),
|
||||
'note':'Subsequent runs use this digest, never silently advance the tag.'})
|
||||
cache=ROOT/'.cache/home';cache.mkdir(parents=True,exist_ok=True)
|
||||
cmd=['docker','run','--rm','--platform=linux/amd64','--user','0:0',
|
||||
'-e','HOME=/work/.cache/home','-v',str(ROOT)+':/work','-w','/work']
|
||||
write_json(lock,{'image':image,'image_id':info['Id'],'resolved_from':IMAGE_TAG,'locked_at':stamp(),'note':'Subsequent runs use this digest, never silently advance the tag.'})
|
||||
cmd=['docker','run','--rm','--platform=linux/amd64','--user','0:0','-e','HOME=/work/.cache/home',
|
||||
'-e','KAM_BUILD_ID='+build_id,'-v',str(ROOT)+':/work','-w','/work']
|
||||
if os.environ.get('GITHUB_TOKEN'):cmd+=['-e','GITHUB_TOKEN']
|
||||
if a.assets:
|
||||
resource=Path(a.assets).resolve()
|
||||
if not resource.is_dir():raise ProbeError('Asset directory not found: '+str(resource))
|
||||
cmd+=['-v',str(resource)+':/probe-assets:ro']
|
||||
cmd+=['--entrypoint','python3',image,'tools/runner.py','--local','--cpu',a.cpu,
|
||||
'--profiles',a.profiles,'--timeout',str(a.timeout)]
|
||||
cmd+=['--entrypoint','python3',image,'tools/runner.py','--local','--cpu',a.cpu,'--timeout',str(a.timeout)]
|
||||
if a.no_fetch:cmd+=['--no-fetch']
|
||||
if a.package_all:cmd+=['--package-all']
|
||||
if a.assets:cmd+=['--assets','/probe-assets']
|
||||
for rel in a.asset_path or []:cmd+=['--asset-path',rel]
|
||||
write_json(out/'docker-command.json',{'command':cmd,'image':image})
|
||||
# Stream the user's real build directly. No background processes/promises.
|
||||
latest=ROOT/'reports/LATEST.json'
|
||||
before=latest.read_bytes() if latest.exists() else None
|
||||
code=subprocess.call(cmd,cwd=ROOT)
|
||||
after=latest.read_bytes() if latest.exists() else None
|
||||
if code in (0,1,2) and after is not None and after!=before:
|
||||
fresh=json.loads(after)
|
||||
if (ROOT/fresh['directory']/'result.json').is_file():return code
|
||||
raise ProbeError('Container could not finish the report; Docker exit code '+str(code))
|
||||
except (ProbeError,OSError,ValueError,KeyError,subprocess.CalledProcessError) as e:
|
||||
for r in result['profiles']:r['compile_link']['reason']=str(e)
|
||||
return finalize(result,out)
|
||||
if code not in (0,1,2):raise ProbeError('Docker exit code '+str(code))
|
||||
record=json.loads(CURRENT_BUILD_METADATA.read_text())
|
||||
if not _valid_docker_handoff(record,build_id,code):raise ProbeError('Container current build metadata is missing, stale, or inconsistent')
|
||||
return code
|
||||
except (ProbeError,OSError,ValueError,KeyError,subprocess.CalledProcessError) as error:
|
||||
return _finish(_record(build_id,a.cpu,'BLOCKED',None,None,compile,package,error=str(error)))
|
||||
|
||||
def main()->int:
|
||||
ap=argparse.ArgumentParser(description=__doc__)
|
||||
ap.add_argument('--local',action='store_true',help='Use installed toolchain; still compile Android only')
|
||||
ap.add_argument('--cpu',choices=CPUS,default='aarch64')
|
||||
ap.add_argument('--profiles',default='all',help='all or comma-separated IDs from config/profiles.json')
|
||||
ap.add_argument('--no-fetch',action='store_true',help='Require previously fetched/verified source tree')
|
||||
ap.add_argument('--assets',help='Optional private KaM resource root; no resources are downloaded')
|
||||
ap.add_argument('--asset-path',action='append',help='Relative path under --assets to stage. Repeat for a small selected fixture set.')
|
||||
ap.add_argument('--package-all',action='store_true',help='Package every successfully linked probe for device startup checks')
|
||||
ap.add_argument('--timeout',type=int,default=1800,help='Per-stage safety timeout in seconds')
|
||||
a=ap.parse_args()
|
||||
if a.asset_path and not a.assets:ap.error('--asset-path requires --assets')
|
||||
if a.timeout<1:ap.error('--timeout must be positive')
|
||||
try:profiles=selected_profiles(a.profiles)
|
||||
except ProbeError as e:ap.error(str(e))
|
||||
return build_local(a,profiles) if a.local else docker_build(a,profiles)
|
||||
inherited=os.environ.get('KAM_BUILD_ID')
|
||||
if inherited and not re.fullmatch(r'\d{8}T\d{6}Z-[0-9a-f]{8}',inherited):ap.error('Invalid KAM_BUILD_ID')
|
||||
build_id=inherited or stamp()+'-'+uuid.uuid4().hex[:8]
|
||||
return build_local(a,build_id) if a.local else docker_build(a,build_id)
|
||||
|
||||
if __name__=='__main__':raise SystemExit(main())
|
||||
Reference in new issue
Block a user