"""One-command Android-only compile, full native link, APK and evidence pipeline.""" from __future__ import annotations import argparse, json, os, re, shutil, subprocess, uuid, zipfile from pathlib import Path from common import (ROOT, COMMIT, CPUS, ProbeError, stamp, sha256, write_json, run_command, first_errors, CURRENT_OUTPUT_DIR, CURRENT_BUILD_METADATA, CURRENT_PACKAGE_METADATA, CURRENT_APK, CURRENT_BUILD_SCHEMA, CURRENT_PACKAGE_SCHEMA) from source_fetch import fetch, verify from source_audit import inventory from source_resolution import load_source_contract, validate_source_contract, build_source_resolution from generate import materialize_application_project from artifacts import inspect_elf, inspect_apk from resource_staging import stage_runtime_resources IMAGE_TAG='kambi/castle-engine-cloud-builds-tools:cge-unstable' def cge_command(cpu:str,action:str)->list[str]: if cpu not in CPUS:raise ProbeError('Unsupported Android CPU') if action not in ('compile','package'):raise ProbeError('Not an Android build action') cmd=['castle-engine','--os=android','--cpu='+cpu,'--compiler=fpc','--mode=debug'] if action=='package':cmd+=['--assume-compiled','--package-format=android-apk'] return cmd+[action] def tool_versions(out:Path)->dict: commands={'castle':['castle-engine','--version'],'fpc':['fpc','-iV'],'java':['java','-version'], 'adb':['adb','version']} d={} for name,cmd in commands.items(): if shutil.which(cmd[0]): rec=run_command(cmd,ROOT,out/(name+'-version.log'),60) rec['output']=(out/(name+'-version.log')).read_text(errors='replace')[:16000];d[name]=rec else:d[name]={'status':'NOT_INSTALLED'} d['paths']={k:os.environ[k] for k in ('CASTLE_ENGINE_PATH','ANDROID_HOME','ANDROID_SDK_ROOT', 'ANDROID_NDK_ROOT','JAVA_HOME','FPCDIR') if k in os.environ} for variable in ('ANDROID_NDK_ROOT','ANDROID_HOME','ANDROID_SDK_ROOT'): if os.environ.get(variable): p=Path(os.environ[variable])/'source.properties' if p.is_file():d[variable+'_source_properties']=p.read_text(errors='replace') lock=ROOT/'config/toolchain.lock.json' if lock.is_file():d['docker_image_lock']=json.loads(lock.read_text()) write_json(out/'toolchain.json',d) return d def signature_tool()->str|None: direct=shutil.which('apksigner') if direct:return direct paths=[] for key in ('ANDROID_HOME','ANDROID_SDK_ROOT'): if os.environ.get(key):paths+=list(Path(os.environ[key]).glob('build-tools/*/apksigner')) return str(sorted(paths)[-1]) if paths else None def _application(descriptor:dict|None)->dict|None: if descriptor is None:return None return {key:descriptor[key] for key in ('name','caption','package','version','version_code','game_unit')} def _record(build_id:str, cpu:str, status:str, application:dict|None=None, resource_count:int|None=None, compile:dict|None=None, package:dict|None=None, package_metadata:str|None=None, error:str|None=None)->dict: return {'schema':CURRENT_BUILD_SCHEMA,'build_id':build_id,'status':status,'cpu':cpu,'abi':CPUS[cpu][0], 'application':application,'upstream_commit':COMMIT,'resource_count':resource_count, 'compile':compile or {'status':'NOT_RUN','exit_code':None}, 'package':package or {'status':'NOT_RUN','exit_code':None}, 'package_metadata':package_metadata,'error':error} def _finish(record:dict)->int: if record['status']!='PASS': CURRENT_OUTPUT_DIR.mkdir(parents=True,exist_ok=True) for path in (CURRENT_PACKAGE_METADATA,CURRENT_APK): path.unlink(missing_ok=True) record['package_metadata']=None write_json(CURRENT_BUILD_METADATA,record) return {'PASS':0,'FAIL':1,'BLOCKED':2}[record['status']] def _valid_docker_handoff(record:object,build_id:str,code:int)->bool: fields={'schema','build_id','status','cpu','abi','application','upstream_commit','resource_count','compile','package','package_metadata','error'} statuses={'PASS':0,'FAIL':1,'BLOCKED':2};stage_statuses=set(statuses)|{'NOT_RUN'} if not isinstance(record,dict) or set(record)!=fields or record.get('schema')!=CURRENT_BUILD_SCHEMA or record.get('build_id')!=build_id or statuses.get(record.get('status'))!=code:return False cpu=record.get('cpu') if cpu not in CPUS or record.get('abi')!=CPUS[cpu][0] or record.get('upstream_commit')!=COMMIT:return False if record['resource_count'] is not None and (type(record['resource_count']) is not int or record['resource_count']<0):return False app=record['application'] if app is not None and (not isinstance(app,dict) or set(app)!={'name','caption','package','version','version_code','game_unit'} or any(not isinstance(app[key],str) or not app[key] for key in ('name','caption','package','version','game_unit')) or type(app['version_code']) is not int):return False for stage in ('compile','package'): value=record[stage] if not isinstance(value,dict) or set(value)!={'status','exit_code'} or value.get('status') not in stage_statuses:return False if (value['status']=='NOT_RUN') != (value['exit_code'] is None) or value['exit_code'] is not None and type(value['exit_code']) is not int:return False if record['status']=='PASS': return app is not None and record['resource_count'] is not None and record['compile']['status']==record['package']['status']=='PASS' and record['package_metadata']=='out/current/package.json' and record['error'] is None return record['package_metadata'] is None and isinstance(record['error'],str) and bool(record['error']) def _error(log:Path)->str: text=log.read_text(errors='replace') if log.exists() else '' return (first_errors(text) or [text[-1500:] or 'Build command failed'])[0] def _native_library_path(cpu:str)->str: return f'lib/{CPUS[cpu][0]}/libkam_mobile.so' def _select_native_library(project:Path, name:str, cpu:str)->Path: candidates=sorted(project.rglob(f'lib{name}_android_{cpu}.so'),key=lambda path:str(path)) if not candidates:raise ProbeError('Compiler exited 0 but expected native library is missing') hashes={sha256(path) for path in candidates} if len(hashes)!=1:raise ProbeError('Ambiguous native library outputs for '+CPUS[cpu][0]+': '+', '.join(str(path) for path in candidates)) return candidates[0] def _prepare(build_id:str, diagnostics:bool=True)->tuple[Path,Path]: work=ROOT/'build'/build_id; logs=work/'logs' logs.mkdir(parents=True,exist_ok=True) if diagnostics:tool_versions(logs) return work,logs def build_local(a:argparse.Namespace,build_id:str)->int: work,logs=_prepare(build_id, not bool(os.environ.get('KAM_BUILD_ID'))) app=None; resources=None; compile={'status':'NOT_RUN','exit_code':None}; package={'status':'NOT_RUN','exit_code':None} try: vendor_root=ROOT/'vendor/kam_remake' receipt=verify(vendor_root) if (vendor_root/'SOURCE_RECEIPT.json').is_file() else (fetch(vendor_root) if not a.no_fetch else (_ for _ in ()).throw(ProbeError('Pinned source tree absent and --no-fetch requested'))) audited=inventory(vendor_root) validated=validate_source_contract(load_source_contract(),vendor_root,receipt,audited) resolution=build_source_resolution(validated,vendor_root,ROOT/'code') descriptor=materialize_application_project(work/'application',resolution);app=_application(descriptor) resources=stage_runtime_resources(descriptor['data_root']) except (ProbeError,OSError,ValueError,KeyError) as error: return _finish(_record(build_id,a.cpu,'BLOCKED',app,None,compile,package,error=str(error))) if not shutil.which('castle-engine'): return _finish(_record(build_id,a.cpu,'BLOCKED',app,len(resources),compile,package,error='castle-engine not installed')) project=descriptor['project_dir']; compile_result=run_command(cge_command(a.cpu,'compile'),project,logs/'compile.log',a.timeout) compile={'status':compile_result['status'],'exit_code':compile_result['exit_code']} if compile['status']!='PASS': return _finish(_record(build_id,a.cpu,compile['status'],app,len(resources),compile,package,error=_error(logs/'compile.log'))) try: library=_select_native_library(project,descriptor['name'],a.cpu); inspect_elf(library.read_bytes(),a.cpu) except (ProbeError,OSError,ValueError) as error: compile['status']='FAIL' return _finish(_record(build_id,a.cpu,'FAIL',app,len(resources),compile,package,error=str(error))) package_result=run_command(cge_command(a.cpu,'package'),project,logs/'package.log',a.timeout) package={'status':package_result['status'],'exit_code':package_result['exit_code']} if package['status']!='PASS': return _finish(_record(build_id,a.cpu,package['status'],app,len(resources),compile,package,error=_error(logs/'package.log'))) try: candidates=list(project.rglob('*.apk')) if not candidates:raise ProbeError('Packaging exited 0 but no APK') if len({sha256(path) for path in candidates})!=1:raise ProbeError('Multiple different APK outputs') apk=candidates[0]; apk_info=inspect_apk(apk,a.cpu,library.name,sha256(library),library.read_bytes()) if apk_info['main_library_path']!=_native_library_path(a.cpu):raise ProbeError('APK native library path does not match selected ABI and libkam_mobile.so') signer=signature_tool() if signer: signed=run_command([signer,'verify','--verbose',str(apk)],project,logs/'signature.log',a.timeout) if signed['status']!='PASS':raise ProbeError(_error(logs/'signature.log')) except (ProbeError,OSError,ValueError,KeyError,zipfile.BadZipFile) as error: package['status']='FAIL' return _finish(_record(build_id,a.cpu,'FAIL',app,len(resources),compile,package,error=str(error))) try: CURRENT_OUTPUT_DIR.mkdir(parents=True,exist_ok=True) temporary=CURRENT_APK.with_name(CURRENT_APK.name+'.tmp') shutil.copyfile(apk,temporary);os.replace(temporary,CURRENT_APK) package_data={'schema':CURRENT_PACKAGE_SCHEMA,'build_id':build_id, 'application':{key:app[key] for key in ('name','package','version','version_code')}, 'cpu':a.cpu,'abi':CPUS[a.cpu][0], 'apk':{'file':'out/current/kam-mobile.apk','sha256':sha256(CURRENT_APK),'size':CURRENT_APK.stat().st_size, 'native_library_path_in_apk':apk_info['main_library_path']}} write_json(CURRENT_PACKAGE_METADATA,package_data) return _finish(_record(build_id,a.cpu,'PASS',app,len(resources),compile,package, 'out/current/package.json',None)) except (OSError,ValueError,KeyError) as error: package['status']='FAIL' return _finish(_record(build_id,a.cpu,'FAIL',app,len(resources),compile,package,error=str(error))) def docker_build(a:argparse.Namespace,build_id:str)->int: compile={'status':'NOT_RUN','exit_code':None};package={'status':'NOT_RUN','exit_code':None} try: work,logs=_prepare(build_id) if not shutil.which('docker'):raise ProbeError('Docker not installed') lock=ROOT/'config/toolchain.lock.json' if lock.is_file(): image=json.loads(lock.read_text())['image'] if not re.fullmatch(r'[\w./-]+@sha256:[a-f0-9]{64}',image):raise ProbeError('Invalid toolchain digest lock') if subprocess.run(['docker','image','inspect',image],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL).returncode: if run_command(['docker','pull','--platform=linux/amd64',image],ROOT,logs/'docker-pull.log',a.timeout)['status']!='PASS':raise ProbeError('Cannot obtain locked Docker image') else: if run_command(['docker','pull','--platform=linux/amd64',IMAGE_TAG],ROOT,logs/'docker-pull.log',a.timeout)['status']!='PASS':raise ProbeError('Docker image pull failed') info=json.loads(subprocess.check_output(['docker','image','inspect',IMAGE_TAG],text=True))[0] image=next((value for value in info.get('RepoDigests',[]) if 'castle-engine-cloud-builds-tools@sha256:' in value),None) if image is None:raise ProbeError('Image has no immutable RepoDigest') write_json(lock,{'image':image,'image_id':info['Id'],'resolved_from':IMAGE_TAG,'locked_at':stamp(),'note':'Subsequent runs use this digest, never silently advance the tag.'}) cmd=['docker','run','--rm','--platform=linux/amd64','--user','0:0','-e','HOME=/work/.cache/home', '-e','KAM_BUILD_ID='+build_id,'-v',str(ROOT)+':/work','-w','/work'] if os.environ.get('GITHUB_TOKEN'):cmd+=['-e','GITHUB_TOKEN'] cmd+=['--entrypoint','python3',image,'tools/runner.py','--local','--cpu',a.cpu,'--timeout',str(a.timeout)] if a.no_fetch:cmd+=['--no-fetch'] code=subprocess.call(cmd,cwd=ROOT) if code not in (0,1,2):raise ProbeError('Docker exit code '+str(code)) record=json.loads(CURRENT_BUILD_METADATA.read_text()) if not _valid_docker_handoff(record,build_id,code):raise ProbeError('Container current build metadata is missing, stale, or inconsistent') return code except (ProbeError,OSError,ValueError,KeyError,subprocess.CalledProcessError) as error: return _finish(_record(build_id,a.cpu,'BLOCKED',None,None,compile,package,error=str(error))) def main()->int: ap=argparse.ArgumentParser(description=__doc__) ap.add_argument('--local',action='store_true',help='Use installed toolchain; still compile Android only') ap.add_argument('--cpu',choices=CPUS,default='aarch64') ap.add_argument('--no-fetch',action='store_true',help='Require previously fetched/verified source tree') ap.add_argument('--timeout',type=int,default=1800,help='Per-stage safety timeout in seconds') a=ap.parse_args() if a.timeout<1:ap.error('--timeout must be positive') inherited=os.environ.get('KAM_BUILD_ID') if inherited and not re.fullmatch(r'\d{8}T\d{6}Z-[0-9a-f]{8}',inherited):ap.error('Invalid KAM_BUILD_ID') build_id=inherited or stamp()+'-'+uuid.uuid4().hex[:8] return build_local(a,build_id) if a.local else docker_build(a,build_id) if __name__=='__main__':raise SystemExit(main())