248 lines
16 KiB
Python
Executable File
248 lines
16 KiB
Python
Executable File
"""One-command Android-only compile, full native link, APK and evidence pipeline."""
|
|
from __future__ import annotations
|
|
import argparse, json, os, re, shutil, subprocess, uuid, zipfile
|
|
from pathlib import Path
|
|
from common import (ROOT, COMMIT, CPUS, ProbeError, stamp, sha256, write_json, run_command,
|
|
first_errors, CURRENT_OUTPUT_DIR, CURRENT_BUILD_METADATA,
|
|
CURRENT_PACKAGE_METADATA, CURRENT_APK, CURRENT_BUILD_SCHEMA,
|
|
CURRENT_PACKAGE_SCHEMA)
|
|
from source_fetch import fetch, verify
|
|
from source_audit import inventory
|
|
from source_resolution import load_source_contract, validate_source_contract, build_source_resolution
|
|
from generate import android_service_roots, materialize_application_project
|
|
from artifacts import inspect_elf, inspect_apk
|
|
from resource_staging import stage_runtime_resources
|
|
|
|
IMAGE_TAG='kambi/castle-engine-cloud-builds-tools:cge-unstable'
|
|
BUILD_EVIDENCE_DIR=ROOT/'build-evidence'
|
|
def cge_command(cpu:str,action:str)->list[str]:
|
|
if cpu not in CPUS:raise ProbeError('Unsupported Android CPU')
|
|
if action not in ('compile','package'):raise ProbeError('Not an Android build action')
|
|
cmd=['castle-engine','--os=android','--cpu='+cpu,'--compiler=fpc','--mode=debug']
|
|
if action=='package':cmd+=['--assume-compiled','--package-format=android-apk']
|
|
return cmd+[action]
|
|
|
|
def tool_versions(out:Path)->dict:
|
|
commands={'castle':['castle-engine','--version'],'fpc':['fpc','-iV'],'java':['java','-version'],
|
|
'adb':['adb','version']}
|
|
d={}
|
|
for name,cmd in commands.items():
|
|
if shutil.which(cmd[0]):
|
|
rec=run_command(cmd,ROOT,out/(name+'-version.log'),60)
|
|
rec['output']=(out/(name+'-version.log')).read_text(errors='replace')[:16000];d[name]=rec
|
|
else:d[name]={'status':'NOT_INSTALLED'}
|
|
d['paths']={k:os.environ[k] for k in ('CASTLE_ENGINE_PATH','ANDROID_HOME','ANDROID_SDK_ROOT',
|
|
'ANDROID_NDK_ROOT','JAVA_HOME','FPCDIR') if k in os.environ}
|
|
for variable in ('ANDROID_NDK_ROOT','ANDROID_HOME','ANDROID_SDK_ROOT'):
|
|
if os.environ.get(variable):
|
|
p=Path(os.environ[variable])/'source.properties'
|
|
if p.is_file():d[variable+'_source_properties']=p.read_text(errors='replace')
|
|
lock=ROOT/'config/toolchain.lock.json'
|
|
if lock.is_file():d['docker_image_lock']=json.loads(lock.read_text())
|
|
write_json(out/'toolchain.json',d)
|
|
return d
|
|
|
|
def signature_tool()->str|None:
|
|
direct=shutil.which('apksigner')
|
|
if direct:return direct
|
|
paths=[]
|
|
for key in ('ANDROID_HOME','ANDROID_SDK_ROOT'):
|
|
if os.environ.get(key):paths+=list(Path(os.environ[key]).glob('build-tools/*/apksigner'))
|
|
return str(sorted(paths)[-1]) if paths else None
|
|
|
|
def _application(descriptor:dict|None)->dict|None:
|
|
if descriptor is None:return None
|
|
return {key:descriptor[key] for key in ('name','caption','package','version','version_code','game_unit')}
|
|
|
|
def _record(build_id:str, cpu:str, status:str, application:dict|None=None,
|
|
resource_count:int|None=None, compile:dict|None=None, package:dict|None=None,
|
|
package_metadata:str|None=None, error:str|None=None)->dict:
|
|
return {'schema':CURRENT_BUILD_SCHEMA,'build_id':build_id,'status':status,'cpu':cpu,'abi':CPUS[cpu][0],
|
|
'application':application,'upstream_commit':COMMIT,'resource_count':resource_count,
|
|
'compile':compile or {'status':'NOT_RUN','exit_code':None},
|
|
'package':package or {'status':'NOT_RUN','exit_code':None},
|
|
'package_metadata':package_metadata,'error':error}
|
|
|
|
def _publish_build_evidence(record:dict)->None:
|
|
source=ROOT/'build'/record['build_id']/'logs'
|
|
destination=BUILD_EVIDENCE_DIR/record['build_id']/'logs'
|
|
destination.mkdir(parents=True,exist_ok=True)
|
|
if source.is_dir():
|
|
for path in source.iterdir():
|
|
if path.is_file():shutil.copy2(path,destination/path.name)
|
|
write_json(destination.parent/'build.json',record)
|
|
|
|
def _finish(record:dict)->int:
|
|
if record['status']!='PASS':
|
|
CURRENT_OUTPUT_DIR.mkdir(parents=True,exist_ok=True)
|
|
for path in (CURRENT_PACKAGE_METADATA,CURRENT_APK):
|
|
path.unlink(missing_ok=True)
|
|
record['package_metadata']=None
|
|
write_json(CURRENT_BUILD_METADATA,record)
|
|
_publish_build_evidence(record)
|
|
return {'PASS':0,'FAIL':1,'BLOCKED':2}[record['status']]
|
|
|
|
def _valid_docker_handoff(record:object,build_id:str,code:int)->bool:
|
|
fields={'schema','build_id','status','cpu','abi','application','upstream_commit','resource_count','compile','package','package_metadata','error'}
|
|
statuses={'PASS':0,'FAIL':1,'BLOCKED':2};stage_statuses=set(statuses)|{'NOT_RUN'}
|
|
if not isinstance(record,dict) or set(record)!=fields or record.get('schema')!=CURRENT_BUILD_SCHEMA or record.get('build_id')!=build_id or statuses.get(record.get('status'))!=code:return False
|
|
cpu=record.get('cpu')
|
|
if cpu not in CPUS or record.get('abi')!=CPUS[cpu][0] or record.get('upstream_commit')!=COMMIT:return False
|
|
if record['resource_count'] is not None and (type(record['resource_count']) is not int or record['resource_count']<0):return False
|
|
app=record['application']
|
|
if app is not None and (not isinstance(app,dict) or set(app)!={'name','caption','package','version','version_code','game_unit'} or any(not isinstance(app[key],str) or not app[key] for key in ('name','caption','package','version','game_unit')) or type(app['version_code']) is not int):return False
|
|
for stage in ('compile','package'):
|
|
value=record[stage]
|
|
if not isinstance(value,dict) or set(value)!={'status','exit_code'} or value.get('status') not in stage_statuses:return False
|
|
if (value['status']=='NOT_RUN') != (value['exit_code'] is None) or value['exit_code'] is not None and type(value['exit_code']) is not int:return False
|
|
if record['status']=='PASS':
|
|
return app is not None and record['resource_count'] is not None and record['compile']['status']==record['package']['status']=='PASS' and record['package_metadata']=='out/current/package.json' and record['error'] is None
|
|
return record['package_metadata'] is None and isinstance(record['error'],str) and bool(record['error'])
|
|
|
|
def _error(log:Path)->str:
|
|
text=log.read_text(errors='replace') if log.exists() else ''
|
|
return (first_errors(text) or [text[-1500:] or 'Build command failed'])[0]
|
|
|
|
def _native_library_path(cpu:str)->str:
|
|
return f'lib/{CPUS[cpu][0]}/libkam_mobile.so'
|
|
|
|
def _select_native_library(project:Path, name:str, cpu:str)->Path:
|
|
candidates=sorted(project.rglob(f'lib{name}_android_{cpu}.so'),key=lambda path:str(path))
|
|
if not candidates:raise ProbeError('Compiler exited 0 but expected native library is missing')
|
|
hashes={sha256(path) for path in candidates}
|
|
if len(hashes)!=1:raise ProbeError('Ambiguous native library outputs for '+CPUS[cpu][0]+': '+', '.join(str(path) for path in candidates))
|
|
return candidates[0]
|
|
|
|
def _prepare(build_id:str, diagnostics:bool=True)->tuple[Path,Path]:
|
|
work=ROOT/'build'/build_id; logs=work/'logs'
|
|
logs.mkdir(parents=True,exist_ok=True)
|
|
if diagnostics:tool_versions(logs)
|
|
return work,logs
|
|
|
|
def stage_android_services(project:Path, logs:Path)->None:
|
|
services=android_service_roots(project/'android_services')
|
|
if not services:return
|
|
if os.environ.get('KAM_CONTAINER_BUILD')!='1':
|
|
raise ProbeError('Project-owned Android services require the normal disposable-container build: ./build-android.sh')
|
|
castle_path=os.environ.get('CASTLE_ENGINE_PATH')
|
|
if not castle_path:
|
|
raise ProbeError('Cannot stage project-owned Android services: CASTLE_ENGINE_PATH is not set')
|
|
service_root=Path(castle_path)/'tools/build-tool/data/android/services'
|
|
if not service_root.is_dir():
|
|
raise ProbeError('Cannot stage project-owned Android services: Castle service root is missing: '+str(service_root))
|
|
try:
|
|
with (logs/'android-services.log').open('w',encoding='utf-8') as log:
|
|
for service in services:
|
|
destination=service_root/service.name
|
|
if destination.exists() or destination.is_symlink():
|
|
raise ProbeError('Project-owned Android service collides with Castle built-in service: '+service.name)
|
|
shutil.copytree(service,destination)
|
|
log.write(f'project service: {service.name}\nsource: {service}\ncastle destination: {destination}\nstatus: staged\n')
|
|
except ProbeError:
|
|
raise
|
|
except OSError as error:
|
|
raise ProbeError(f'Cannot stage project-owned Android services: {error}') from error
|
|
|
|
def build_local(a:argparse.Namespace,build_id:str)->int:
|
|
work,logs=_prepare(build_id, not bool(os.environ.get('KAM_BUILD_ID')))
|
|
app=None; resources=None; compile={'status':'NOT_RUN','exit_code':None}; package={'status':'NOT_RUN','exit_code':None}
|
|
try:
|
|
vendor_root=ROOT/'vendor/kam_remake'
|
|
receipt=verify(vendor_root) if (vendor_root/'SOURCE_RECEIPT.json').is_file() else (fetch(vendor_root) if not a.no_fetch else (_ for _ in ()).throw(ProbeError('Pinned source tree absent and --no-fetch requested')))
|
|
audited=inventory(vendor_root)
|
|
validated=validate_source_contract(load_source_contract(),vendor_root,receipt,audited)
|
|
resolution=build_source_resolution(validated,vendor_root,ROOT/'code')
|
|
descriptor=materialize_application_project(work/'application',resolution);app=_application(descriptor)
|
|
resources=stage_runtime_resources(descriptor['data_root'])
|
|
stage_android_services(descriptor['project_dir'],logs)
|
|
except (ProbeError,OSError,ValueError,KeyError) as error:
|
|
return _finish(_record(build_id,a.cpu,'BLOCKED',app,None,compile,package,error=str(error)))
|
|
if not shutil.which('castle-engine'):
|
|
return _finish(_record(build_id,a.cpu,'BLOCKED',app,len(resources),compile,package,error='castle-engine not installed'))
|
|
project=descriptor['project_dir']; compile_result=run_command(cge_command(a.cpu,'compile'),project,logs/'compile.log',a.timeout)
|
|
compile={'status':compile_result['status'],'exit_code':compile_result['exit_code']}
|
|
if compile['status']!='PASS':
|
|
return _finish(_record(build_id,a.cpu,compile['status'],app,len(resources),compile,package,error=_error(logs/'compile.log')))
|
|
try:
|
|
library=_select_native_library(project,descriptor['name'],a.cpu); inspect_elf(library.read_bytes(),a.cpu)
|
|
except (ProbeError,OSError,ValueError) as error:
|
|
compile['status']='FAIL'
|
|
return _finish(_record(build_id,a.cpu,'FAIL',app,len(resources),compile,package,error=str(error)))
|
|
package_result=run_command(cge_command(a.cpu,'package'),project,logs/'package.log',a.timeout)
|
|
package={'status':package_result['status'],'exit_code':package_result['exit_code']}
|
|
if package['status']!='PASS':
|
|
return _finish(_record(build_id,a.cpu,package['status'],app,len(resources),compile,package,error=_error(logs/'package.log')))
|
|
try:
|
|
candidates=list(project.rglob('*.apk'))
|
|
if not candidates:raise ProbeError('Packaging exited 0 but no APK')
|
|
if len({sha256(path) for path in candidates})!=1:raise ProbeError('Multiple different APK outputs')
|
|
apk=candidates[0]; apk_info=inspect_apk(apk,a.cpu,library.name,sha256(library),library.read_bytes())
|
|
if apk_info['main_library_path']!=_native_library_path(a.cpu):raise ProbeError('APK native library path does not match selected ABI and libkam_mobile.so')
|
|
signer=signature_tool()
|
|
if signer:
|
|
signed=run_command([signer,'verify','--verbose',str(apk)],project,logs/'signature.log',a.timeout)
|
|
if signed['status']!='PASS':raise ProbeError(_error(logs/'signature.log'))
|
|
except (ProbeError,OSError,ValueError,KeyError,zipfile.BadZipFile) as error:
|
|
package['status']='FAIL'
|
|
return _finish(_record(build_id,a.cpu,'FAIL',app,len(resources),compile,package,error=str(error)))
|
|
try:
|
|
CURRENT_OUTPUT_DIR.mkdir(parents=True,exist_ok=True)
|
|
temporary=CURRENT_APK.with_name(CURRENT_APK.name+'.tmp')
|
|
shutil.copyfile(apk,temporary);os.replace(temporary,CURRENT_APK)
|
|
package_data={'schema':CURRENT_PACKAGE_SCHEMA,'build_id':build_id,
|
|
'application':{key:app[key] for key in ('name','package','version','version_code')},
|
|
'cpu':a.cpu,'abi':CPUS[a.cpu][0],
|
|
'apk':{'file':'out/current/kam-mobile.apk','sha256':sha256(CURRENT_APK),'size':CURRENT_APK.stat().st_size,
|
|
'native_library_path_in_apk':apk_info['main_library_path']}}
|
|
write_json(CURRENT_PACKAGE_METADATA,package_data)
|
|
return _finish(_record(build_id,a.cpu,'PASS',app,len(resources),compile,package,
|
|
'out/current/package.json',None))
|
|
except (OSError,ValueError,KeyError) as error:
|
|
package['status']='FAIL'
|
|
return _finish(_record(build_id,a.cpu,'FAIL',app,len(resources),compile,package,error=str(error)))
|
|
|
|
def docker_build(a:argparse.Namespace,build_id:str)->int:
|
|
compile={'status':'NOT_RUN','exit_code':None};package={'status':'NOT_RUN','exit_code':None}
|
|
try:
|
|
work,logs=_prepare(build_id)
|
|
if not shutil.which('docker'):raise ProbeError('Docker not installed')
|
|
lock=ROOT/'config/toolchain.lock.json'
|
|
if lock.is_file():
|
|
image=json.loads(lock.read_text())['image']
|
|
if not re.fullmatch(r'[\w./-]+@sha256:[a-f0-9]{64}',image):raise ProbeError('Invalid toolchain digest lock')
|
|
if subprocess.run(['docker','image','inspect',image],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL).returncode:
|
|
if run_command(['docker','pull','--platform=linux/amd64',image],ROOT,logs/'docker-pull.log',a.timeout)['status']!='PASS':raise ProbeError('Cannot obtain locked Docker image')
|
|
else:
|
|
if run_command(['docker','pull','--platform=linux/amd64',IMAGE_TAG],ROOT,logs/'docker-pull.log',a.timeout)['status']!='PASS':raise ProbeError('Docker image pull failed')
|
|
info=json.loads(subprocess.check_output(['docker','image','inspect',IMAGE_TAG],text=True))[0]
|
|
image=next((value for value in info.get('RepoDigests',[]) if 'castle-engine-cloud-builds-tools@sha256:' in value),None)
|
|
if image is None:raise ProbeError('Image has no immutable RepoDigest')
|
|
write_json(lock,{'image':image,'image_id':info['Id'],'resolved_from':IMAGE_TAG,'locked_at':stamp(),'note':'Subsequent runs use this digest, never silently advance the tag.'})
|
|
cmd=['docker','run','--rm','--platform=linux/amd64','--user','0:0','-e','HOME=/work/.cache/home',
|
|
'-e','KAM_BUILD_ID='+build_id,'-e','KAM_CONTAINER_BUILD=1','-v',str(ROOT)+':/work','-w','/work']
|
|
if os.environ.get('GITHUB_TOKEN'):cmd+=['-e','GITHUB_TOKEN']
|
|
cmd+=['--entrypoint','python3',image,'tools/runner.py','--local','--cpu',a.cpu,'--timeout',str(a.timeout)]
|
|
if a.no_fetch:cmd+=['--no-fetch']
|
|
code=subprocess.call(cmd,cwd=ROOT)
|
|
if code not in (0,1,2):raise ProbeError('Docker exit code '+str(code))
|
|
record=json.loads(CURRENT_BUILD_METADATA.read_text())
|
|
if not _valid_docker_handoff(record,build_id,code):raise ProbeError('Container current build metadata is missing, stale, or inconsistent')
|
|
return code
|
|
except (ProbeError,OSError,ValueError,KeyError,subprocess.CalledProcessError) as error:
|
|
return _finish(_record(build_id,a.cpu,'BLOCKED',None,None,compile,package,error=str(error)))
|
|
|
|
def main()->int:
|
|
ap=argparse.ArgumentParser(description=__doc__)
|
|
ap.add_argument('--local',action='store_true',help='Use installed toolchain; still compile Android only')
|
|
ap.add_argument('--cpu',choices=CPUS,default='aarch64')
|
|
ap.add_argument('--no-fetch',action='store_true',help='Require previously fetched/verified source tree')
|
|
ap.add_argument('--timeout',type=int,default=1800,help='Per-stage safety timeout in seconds')
|
|
a=ap.parse_args()
|
|
if a.timeout<1:ap.error('--timeout must be positive')
|
|
inherited=os.environ.get('KAM_BUILD_ID')
|
|
if inherited and not re.fullmatch(r'\d{8}T\d{6}Z-[0-9a-f]{8}',inherited):ap.error('Invalid KAM_BUILD_ID')
|
|
build_id=inherited or stamp()+'-'+uuid.uuid4().hex[:8]
|
|
return build_local(a,build_id) if a.local else docker_build(a,build_id)
|
|
|
|
if __name__=='__main__':raise SystemExit(main())
|