kam-mobile/tools/runner.py

"""One-command Android-only compile, full native link, APK and evidence pipeline."""
from __future__ import annotations
import argparse, json, os, re, shutil, subprocess, uuid, zipfile
from pathlib import Path
from common import (ROOT, COMMIT, CPUS, ProbeError, stamp, sha256, write_json, run_command,
                    first_errors, CURRENT_OUTPUT_DIR, CURRENT_BUILD_METADATA,
                    CURRENT_PACKAGE_METADATA, CURRENT_APK, CURRENT_BUILD_SCHEMA,
                    CURRENT_PACKAGE_SCHEMA)
from source_fetch import fetch, verify
from source_audit import inventory
from source_resolution import load_source_contract, validate_source_contract, build_source_resolution
from generate import android_service_roots, materialize_application_project
from artifacts import inspect_elf, inspect_apk
from resource_staging import stage_runtime_resources

IMAGE_TAG='kambi/castle-engine-cloud-builds-tools:cge-unstable'
def cge_command(cpu:str,action:str)->list[str]:
    if cpu not in CPUS:raise ProbeError('Unsupported Android CPU')
    if action not in ('compile','package'):raise ProbeError('Not an Android build action')
    cmd=['castle-engine','--os=android','--cpu='+cpu,'--compiler=fpc','--mode=debug']
    if action=='package':cmd+=['--assume-compiled','--package-format=android-apk']
    return cmd+[action]

def tool_versions(out:Path)->dict:
    commands={'castle':['castle-engine','--version'],'fpc':['fpc','-iV'],'java':['java','-version'],
              'adb':['adb','version']}
    d={}
    for name,cmd in commands.items():
        if shutil.which(cmd[0]):
            rec=run_command(cmd,ROOT,out/(name+'-version.log'),60)
            rec['output']=(out/(name+'-version.log')).read_text(errors='replace')[:16000];d[name]=rec
        else:d[name]={'status':'NOT_INSTALLED'}
    d['paths']={k:os.environ[k] for k in ('CASTLE_ENGINE_PATH','ANDROID_HOME','ANDROID_SDK_ROOT',
        'ANDROID_NDK_ROOT','JAVA_HOME','FPCDIR') if k in os.environ}
    for variable in ('ANDROID_NDK_ROOT','ANDROID_HOME','ANDROID_SDK_ROOT'):
        if os.environ.get(variable):
            p=Path(os.environ[variable])/'source.properties'
            if p.is_file():d[variable+'_source_properties']=p.read_text(errors='replace')
    lock=ROOT/'config/toolchain.lock.json'
    if lock.is_file():d['docker_image_lock']=json.loads(lock.read_text())
    write_json(out/'toolchain.json',d)
    return d

def signature_tool()->str|None:
    direct=shutil.which('apksigner')
    if direct:return direct
    paths=[]
    for key in ('ANDROID_HOME','ANDROID_SDK_ROOT'):
        if os.environ.get(key):paths+=list(Path(os.environ[key]).glob('build-tools/*/apksigner'))
    return str(sorted(paths)[-1]) if paths else None

def _application(descriptor:dict|None)->dict|None:
    if descriptor is None:return None
    return {key:descriptor[key] for key in ('name','caption','package','version','version_code','game_unit')}

def _record(build_id:str, cpu:str, status:str, application:dict|None=None,
            resource_count:int|None=None, compile:dict|None=None, package:dict|None=None,
            package_metadata:str|None=None, error:str|None=None)->dict:
    return {'schema':CURRENT_BUILD_SCHEMA,'build_id':build_id,'status':status,'cpu':cpu,'abi':CPUS[cpu][0],
            'application':application,'upstream_commit':COMMIT,'resource_count':resource_count,
            'compile':compile or {'status':'NOT_RUN','exit_code':None},
            'package':package or {'status':'NOT_RUN','exit_code':None},
            'package_metadata':package_metadata,'error':error}

def _finish(record:dict)->int:
    if record['status']!='PASS':
        CURRENT_OUTPUT_DIR.mkdir(parents=True,exist_ok=True)
        for path in (CURRENT_PACKAGE_METADATA,CURRENT_APK):
            path.unlink(missing_ok=True)
        record['package_metadata']=None
    write_json(CURRENT_BUILD_METADATA,record)
    return {'PASS':0,'FAIL':1,'BLOCKED':2}[record['status']]

def _valid_docker_handoff(record:object,build_id:str,code:int)->bool:
    fields={'schema','build_id','status','cpu','abi','application','upstream_commit','resource_count','compile','package','package_metadata','error'}
    statuses={'PASS':0,'FAIL':1,'BLOCKED':2};stage_statuses=set(statuses)|{'NOT_RUN'}
    if not isinstance(record,dict) or set(record)!=fields or record.get('schema')!=CURRENT_BUILD_SCHEMA or record.get('build_id')!=build_id or statuses.get(record.get('status'))!=code:return False
    cpu=record.get('cpu')
    if cpu not in CPUS or record.get('abi')!=CPUS[cpu][0] or record.get('upstream_commit')!=COMMIT:return False
    if record['resource_count'] is not None and (type(record['resource_count']) is not int or record['resource_count']<0):return False
    app=record['application']
    if app is not None and (not isinstance(app,dict) or set(app)!={'name','caption','package','version','version_code','game_unit'} or any(not isinstance(app[key],str) or not app[key] for key in ('name','caption','package','version','game_unit')) or type(app['version_code']) is not int):return False
    for stage in ('compile','package'):
        value=record[stage]
        if not isinstance(value,dict) or set(value)!={'status','exit_code'} or value.get('status') not in stage_statuses:return False
        if (value['status']=='NOT_RUN') != (value['exit_code'] is None) or value['exit_code'] is not None and type(value['exit_code']) is not int:return False
    if record['status']=='PASS':
        return app is not None and record['resource_count'] is not None and record['compile']['status']==record['package']['status']=='PASS' and record['package_metadata']=='out/current/package.json' and record['error'] is None
    return record['package_metadata'] is None and isinstance(record['error'],str) and bool(record['error'])

def _error(log:Path)->str:
    text=log.read_text(errors='replace') if log.exists() else ''
    return (first_errors(text) or [text[-1500:] or 'Build command failed'])[0]

def _native_library_path(cpu:str)->str:
    return f'lib/{CPUS[cpu][0]}/libkam_mobile.so'

def _select_native_library(project:Path, name:str, cpu:str)->Path:
    candidates=sorted(project.rglob(f'lib{name}_android_{cpu}.so'),key=lambda path:str(path))
    if not candidates:raise ProbeError('Compiler exited 0 but expected native library is missing')
    hashes={sha256(path) for path in candidates}
    if len(hashes)!=1:raise ProbeError('Ambiguous native library outputs for '+CPUS[cpu][0]+': '+', '.join(str(path) for path in candidates))
    return candidates[0]

def _prepare(build_id:str, diagnostics:bool=True)->tuple[Path,Path]:
    work=ROOT/'build'/build_id; logs=work/'logs'
    logs.mkdir(parents=True,exist_ok=True)
    if diagnostics:tool_versions(logs)
    return work,logs

def stage_android_services(project:Path, logs:Path)->None:
    services=android_service_roots(project/'android_services')
    if not services:return
    if os.environ.get('KAM_CONTAINER_BUILD')!='1':
        raise ProbeError('Project-owned Android services require the normal disposable-container build: ./build-android.sh')
    castle_path=os.environ.get('CASTLE_ENGINE_PATH')
    if not castle_path:
        raise ProbeError('Cannot stage project-owned Android services: CASTLE_ENGINE_PATH is not set')
    service_root=Path(castle_path)/'tools/build-tool/data/android/services'
    if not service_root.is_dir():
        raise ProbeError('Cannot stage project-owned Android services: Castle service root is missing: '+str(service_root))
    try:
        with (logs/'android-services.log').open('w',encoding='utf-8') as log:
            for service in services:
                destination=service_root/service.name
                if destination.exists() or destination.is_symlink():
                    raise ProbeError('Project-owned Android service collides with Castle built-in service: '+service.name)
                shutil.copytree(service,destination)
                log.write(f'project service: {service.name}\nsource: {service}\ncastle destination: {destination}\nstatus: staged\n')
    except ProbeError:
        raise
    except OSError as error:
        raise ProbeError(f'Cannot stage project-owned Android services: {error}') from error

def build_local(a:argparse.Namespace,build_id:str)->int:
    work,logs=_prepare(build_id, not bool(os.environ.get('KAM_BUILD_ID')))
    app=None; resources=None; compile={'status':'NOT_RUN','exit_code':None}; package={'status':'NOT_RUN','exit_code':None}
    try:
        vendor_root=ROOT/'vendor/kam_remake'
        receipt=verify(vendor_root) if (vendor_root/'SOURCE_RECEIPT.json').is_file() else (fetch(vendor_root) if not a.no_fetch else (_ for _ in ()).throw(ProbeError('Pinned source tree absent and --no-fetch requested')))
        audited=inventory(vendor_root)
        validated=validate_source_contract(load_source_contract(),vendor_root,receipt,audited)
        resolution=build_source_resolution(validated,vendor_root,ROOT/'code')
        descriptor=materialize_application_project(work/'application',resolution);app=_application(descriptor)
        resources=stage_runtime_resources(descriptor['data_root'])
        stage_android_services(descriptor['project_dir'],logs)
    except (ProbeError,OSError,ValueError,KeyError) as error:
        return _finish(_record(build_id,a.cpu,'BLOCKED',app,None,compile,package,error=str(error)))
    if not shutil.which('castle-engine'):
        return _finish(_record(build_id,a.cpu,'BLOCKED',app,len(resources),compile,package,error='castle-engine not installed'))
    project=descriptor['project_dir']; compile_result=run_command(cge_command(a.cpu,'compile'),project,logs/'compile.log',a.timeout)
    compile={'status':compile_result['status'],'exit_code':compile_result['exit_code']}
    if compile['status']!='PASS':
        return _finish(_record(build_id,a.cpu,compile['status'],app,len(resources),compile,package,error=_error(logs/'compile.log')))
    try:
        library=_select_native_library(project,descriptor['name'],a.cpu); inspect_elf(library.read_bytes(),a.cpu)
    except (ProbeError,OSError,ValueError) as error:
        compile['status']='FAIL'
        return _finish(_record(build_id,a.cpu,'FAIL',app,len(resources),compile,package,error=str(error)))
    package_result=run_command(cge_command(a.cpu,'package'),project,logs/'package.log',a.timeout)
    package={'status':package_result['status'],'exit_code':package_result['exit_code']}
    if package['status']!='PASS':
        return _finish(_record(build_id,a.cpu,package['status'],app,len(resources),compile,package,error=_error(logs/'package.log')))
    try:
        candidates=list(project.rglob('*.apk'))
        if not candidates:raise ProbeError('Packaging exited 0 but no APK')
        if len({sha256(path) for path in candidates})!=1:raise ProbeError('Multiple different APK outputs')
        apk=candidates[0]; apk_info=inspect_apk(apk,a.cpu,library.name,sha256(library),library.read_bytes())
        if apk_info['main_library_path']!=_native_library_path(a.cpu):raise ProbeError('APK native library path does not match selected ABI and libkam_mobile.so')
        signer=signature_tool()
        if signer:
            signed=run_command([signer,'verify','--verbose',str(apk)],project,logs/'signature.log',a.timeout)
            if signed['status']!='PASS':raise ProbeError(_error(logs/'signature.log'))
    except (ProbeError,OSError,ValueError,KeyError,zipfile.BadZipFile) as error:
        package['status']='FAIL'
        return _finish(_record(build_id,a.cpu,'FAIL',app,len(resources),compile,package,error=str(error)))
    try:
        CURRENT_OUTPUT_DIR.mkdir(parents=True,exist_ok=True)
        temporary=CURRENT_APK.with_name(CURRENT_APK.name+'.tmp')
        shutil.copyfile(apk,temporary);os.replace(temporary,CURRENT_APK)
        package_data={'schema':CURRENT_PACKAGE_SCHEMA,'build_id':build_id,
            'application':{key:app[key] for key in ('name','package','version','version_code')},
            'cpu':a.cpu,'abi':CPUS[a.cpu][0],
            'apk':{'file':'out/current/kam-mobile.apk','sha256':sha256(CURRENT_APK),'size':CURRENT_APK.stat().st_size,
                   'native_library_path_in_apk':apk_info['main_library_path']}}
        write_json(CURRENT_PACKAGE_METADATA,package_data)
        return _finish(_record(build_id,a.cpu,'PASS',app,len(resources),compile,package,
                               'out/current/package.json',None))
    except (OSError,ValueError,KeyError) as error:
        package['status']='FAIL'
        return _finish(_record(build_id,a.cpu,'FAIL',app,len(resources),compile,package,error=str(error)))

def docker_build(a:argparse.Namespace,build_id:str)->int:
    compile={'status':'NOT_RUN','exit_code':None};package={'status':'NOT_RUN','exit_code':None}
    try:
        work,logs=_prepare(build_id)
        if not shutil.which('docker'):raise ProbeError('Docker not installed')
        lock=ROOT/'config/toolchain.lock.json'
        if lock.is_file():
            image=json.loads(lock.read_text())['image']
            if not re.fullmatch(r'[\w./-]+@sha256:[a-f0-9]{64}',image):raise ProbeError('Invalid toolchain digest lock')
            if subprocess.run(['docker','image','inspect',image],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL).returncode:
                if run_command(['docker','pull','--platform=linux/amd64',image],ROOT,logs/'docker-pull.log',a.timeout)['status']!='PASS':raise ProbeError('Cannot obtain locked Docker image')
        else:
            if run_command(['docker','pull','--platform=linux/amd64',IMAGE_TAG],ROOT,logs/'docker-pull.log',a.timeout)['status']!='PASS':raise ProbeError('Docker image pull failed')
            info=json.loads(subprocess.check_output(['docker','image','inspect',IMAGE_TAG],text=True))[0]
            image=next((value for value in info.get('RepoDigests',[]) if 'castle-engine-cloud-builds-tools@sha256:' in value),None)
            if image is None:raise ProbeError('Image has no immutable RepoDigest')
            write_json(lock,{'image':image,'image_id':info['Id'],'resolved_from':IMAGE_TAG,'locked_at':stamp(),'note':'Subsequent runs use this digest, never silently advance the tag.'})
        cmd=['docker','run','--rm','--platform=linux/amd64','--user','0:0','-e','HOME=/work/.cache/home',
             '-e','KAM_BUILD_ID='+build_id,'-e','KAM_CONTAINER_BUILD=1','-v',str(ROOT)+':/work','-w','/work']
        if os.environ.get('GITHUB_TOKEN'):cmd+=['-e','GITHUB_TOKEN']
        cmd+=['--entrypoint','python3',image,'tools/runner.py','--local','--cpu',a.cpu,'--timeout',str(a.timeout)]
        if a.no_fetch:cmd+=['--no-fetch']
        code=subprocess.call(cmd,cwd=ROOT)
        if code not in (0,1,2):raise ProbeError('Docker exit code '+str(code))
        record=json.loads(CURRENT_BUILD_METADATA.read_text())
        if not _valid_docker_handoff(record,build_id,code):raise ProbeError('Container current build metadata is missing, stale, or inconsistent')
        return code
    except (ProbeError,OSError,ValueError,KeyError,subprocess.CalledProcessError) as error:
        return _finish(_record(build_id,a.cpu,'BLOCKED',None,None,compile,package,error=str(error)))

def main()->int:
    ap=argparse.ArgumentParser(description=__doc__)
    ap.add_argument('--local',action='store_true',help='Use installed toolchain; still compile Android only')
    ap.add_argument('--cpu',choices=CPUS,default='aarch64')
    ap.add_argument('--no-fetch',action='store_true',help='Require previously fetched/verified source tree')
    ap.add_argument('--timeout',type=int,default=1800,help='Per-stage safety timeout in seconds')
    a=ap.parse_args()
    if a.timeout<1:ap.error('--timeout must be positive')
    inherited=os.environ.get('KAM_BUILD_ID')
    if inherited and not re.fullmatch(r'\d{8}T\d{6}Z-[0-9a-f]{8}',inherited):ap.error('Invalid KAM_BUILD_ID')
    build_id=inherited or stamp()+'-'+uuid.uuid4().hex[:8]
    return build_local(a,build_id) if a.local else docker_build(a,build_id)

if __name__=='__main__':raise SystemExit(main())