"""One-command Android-only compile, full native link, APK and evidence pipeline.""" from __future__ import annotations import argparse, json, os, re, shutil, subprocess, sys, uuid, zipfile from pathlib import Path from common import ROOT, COMMIT, CPUS, ProbeError, stamp, sha256, write_json, run_command, first_errors, classify from source_fetch import fetch, verify from source_audit import inventory from source_resolution import load_source_contract, validate_source_contract from generate import make_target, asset_inventory from artifacts import inspect_elf, inspect_apk IMAGE_TAG='kambi/castle-engine-cloud-builds-tools:cge-unstable' PROFILES=json.loads((ROOT/'config/profiles.json').read_text()) def selected_profiles(text:str)->list[dict]: if text=='all':return PROFILES names=[s.strip() for s in text.split(',') if s.strip()] available={p['id'] for p in PROFILES} if not names or set(names)-available:raise ProbeError('Unknown/empty profiles: '+text) names=set(names)|{'platform'} # Always establish the same-toolchain platform baseline. return [p for p in PROFILES if p['id'] in names] def cge_command(cpu:str,action:str)->list[str]: if cpu not in CPUS:raise ProbeError('Unsupported Android CPU') if action not in ('compile','package'):raise ProbeError('Not an Android build action') cmd=['castle-engine','--os=android','--cpu='+cpu,'--compiler=fpc','--mode=debug'] if action=='package':cmd+=['--assume-compiled','--package-format=android-apk'] return cmd+[action] def blocked_row(profile:dict,reason:str)->dict: return {'id':profile['id'],'required':profile['required'],'purpose':profile['purpose'], 'compile_link':{'status':'BLOCKED','reason':reason},'package':{'status':'NOT_RUN'}, 'runtime':{'status':'NOT_RUN'},'errors':[]} def gate(rows:list[dict])->str: by_id={r['id']:r for r in rows} required=[p for p in PROFILES if p['required']] if any(r['compile_link']['status']=='FAIL' or r['package']['status']=='FAIL' for r in rows if r['required']): return 'NOT_BUILDABLE_AS_IS' if any(r['compile_link']['status']=='BLOCKED' for r in rows if r['required']):return 'BLOCKED' if any(p['id'] not in by_id for p in required):return 'PARTIAL_SELECTION' for p in required: r=by_id[p['id']] if r['compile_link']['status']!='PASS':return 'BLOCKED' if p['package'] and r['package']['status']!='PASS':return 'BLOCKED' return 'ANDROID_BUILD_GATE_PASS' def render_summary(result:dict,out:Path)->None: title='# KaM r16020 — Android architecture probe\n\n' title+=f"Build ID: `{result['run_id']}` · CPU: `{result['cpu']}` · Gate: **{result['gate']}**\n\n" title+='Нет desktop target. Успех означает сборку выбранных Android-корней, не игровую совместимость.\n\n' title+='| Проба | Compile + native link | APK | Ошибка / следующий шаг |\n|---|---|---|---|\n' for r in result['profiles']: error=(r.get('errors') or [r['compile_link'].get('reason','')])[0] error=error.replace('|','/').replace('\n',' ')[:260] title+=f"| {r['id']} | {r['compile_link']['status']} | {r['package']['status']} | {error} |\n" title+='\n## Что установлено\n\n' title+='`PASS` compile_link = FPC завершился успешно, найден свежий ELF нужного ABI, проверены корневые PPU. Линковка не отключена; требуются определённые символы. ' title+='`PASS` APK = упаковка завершилась, проверены ZIP/DEX, правильный ABI, совпадение native library и DT_NEEDED. ' title+='Подпись проверяется отдельно, если доступен apksigner. `runtime=NOT_RUN` до реального запуска на Android.\n\n' title+='`NOT_BUILDABLE_AS_IS` — найдено препятствие в текущем коде/окружении, а не доказательство, что core надо переписать. ' title+='Ошибки legacy_render/legacy_audio диагностические и не входят в обязательный gate, но их транзитивное попадание в core всё равно видно.\n\n' title+='## Как принимать решение\n\n' title+='| Свидетельство | Разумное решение |\n|---|---|\n' title+='| Корень собирается с оригинальными исходниками | Кандидат на reuse; поведение ещё проверяется отдельно |\n' title+='| Активная зависимость Windows/LCL/desktop GL | Исследовать минимальное отделение платформенной части |\n' title+='| Отсутствует source unit / package | Проверить комплектность исходников, условные defines и пакет для Android |\n' title+='| Undefined reference / отсутствует native library | Сборка/подключение Android-библиотеки либо адаптер Castle |\n' title+='| Один и тот же блокер у многих модулей | Исправить общий корень и повторить пробы; не считать это множеством отдельных переписываний |\n' title+='| Срабатывает exception при запуске | Снять logcat; проверить unit initialization/lifecycle/доступ к данным |\n' title+='\nНи один автоматический результат здесь не выносит решение «написать подсистему с нуля».\n' (out/'SUMMARY.md').write_text(title,encoding='utf-8') def finalize(result:dict,out:Path)->int: result['gate']=gate(result['profiles']) write_json(out/'result.json',result);render_summary(result,out) write_json(ROOT/'reports/LATEST.json',{'run_id':result['run_id'],'directory':str(out.relative_to(ROOT)), 'gate':result['gate'],'summary':str((out/'SUMMARY.md').relative_to(ROOT))}) shutil.copyfile(out/'SUMMARY.md',ROOT/'reports/LATEST-SUMMARY.md') # Evidence intentionally excludes APK/.so, user game assets, caches and source trees. with zipfile.ZipFile(out/'evidence.zip','w',zipfile.ZIP_DEFLATED) as z: for p in sorted(out.rglob('*')): if p.is_file() and p.suffix in ('.json','.log','.md','.txt','.xml','.pas'): z.write(p,p.relative_to(out)) print('\n'+result['gate']) print('Report: '+str(out/'SUMMARY.md')) print('Evidence (no game assets): '+str(out/'evidence.zip')) return 0 if result['gate']=='ANDROID_BUILD_GATE_PASS' else 1 if result['gate']=='NOT_BUILDABLE_AS_IS' else 2 def new_result(cpu:str,profiles:list[dict],reason:str='Not started')->tuple[dict,Path]: rid=stamp()+'-'+uuid.uuid4().hex[:8] out=ROOT/'reports/runs'/rid;out.mkdir(parents=True) return ({'schema':'kam-android-build/1','run_id':rid,'cpu':cpu,'commit':COMMIT, 'desktop_target':False,'shims':False,'runtime':'NOT_RUN', 'profiles':[blocked_row(p,reason) for p in profiles]},out) def tool_versions(out:Path)->dict: commands={'castle':['castle-engine','--version'],'fpc':['fpc','-iV'],'java':['java','-version'], 'adb':['adb','version']} d={} for name,cmd in commands.items(): if shutil.which(cmd[0]): rec=run_command(cmd,ROOT,out/(name+'-version.log'),60) rec['output']=(out/(name+'-version.log')).read_text(errors='replace')[:16000];d[name]=rec else:d[name]={'status':'NOT_INSTALLED'} d['paths']={k:os.environ[k] for k in ('CASTLE_ENGINE_PATH','ANDROID_HOME','ANDROID_SDK_ROOT', 'ANDROID_NDK_ROOT','JAVA_HOME','FPCDIR') if k in os.environ} for variable in ('ANDROID_NDK_ROOT','ANDROID_HOME','ANDROID_SDK_ROOT'): if os.environ.get(variable): p=Path(os.environ[variable])/'source.properties' if p.is_file():d[variable+'_source_properties']=p.read_text(errors='replace') lock=ROOT/'config/toolchain.lock.json' if lock.is_file():d['docker_image_lock']=json.loads(lock.read_text()) write_json(out/'toolchain.json',d) return d def signature_tool()->str|None: direct=shutil.which('apksigner') if direct:return direct paths=[] for key in ('ANDROID_HOME','ANDROID_SDK_ROOT'): if os.environ.get(key):paths+=list(Path(os.environ[key]).glob('build-tools/*/apksigner')) return str(sorted(paths)[-1]) if paths else None def build_local(a:argparse.Namespace,profiles:list[dict])->int: result,out=new_result(a.cpu,profiles) result['toolchain']=tool_versions(out) if not shutil.which('castle-engine'): for r in result['profiles']:r['compile_link']['reason']='castle-engine not installed; no Android compilation attempted' return finalize(result,out) source=ROOT/'vendor/kam_remake';audit=None;validated=None;assets=[] need_source=any(p['units'] for p in profiles) try: contract=load_source_contract() assets=asset_inventory(Path(a.assets),a.asset_path) if a.assets else [] result['asset_count']=len(assets) if need_source: if (source/'SOURCE_RECEIPT.json').is_file():receipt=verify(source) elif a.no_fetch:raise ProbeError('Pinned source tree absent and --no-fetch requested') else:receipt=fetch(source) write_json(out/'source-receipt.json',receipt) audit=inventory(source);write_json(out/'source-inventory.json',audit) validated=validate_source_contract(contract,source,receipt,audit) except (ProbeError,OSError,ValueError,KeyError) as e: result['source_or_inputs_error']=str(e) return finalize(result,out) work=ROOT/'build'/result['run_id'];work.mkdir(parents=True) dist=ROOT/'dist'/result['run_id'];dist.mkdir(parents=True) package_index=[] for idx,profile in enumerate(profiles): row=result['profiles'][idx];ident=profile['id'];project=work/ident print(f"[{idx+1}/{len(profiles)}] Android {a.cpu}: {ident}",flush=True) try: if profile['units']: missing=[u for u in profile['units'] if u.casefold() not in audit['units']] if missing:raise ProbeError('Missing required original source units: '+', '.join(missing)) # Resources only enter APKs when explicitly requested. The platform baseline remains independent. pack=profile['package'] or a.package_all use_assets=bool(a.assets) and pack and ident=='platform' description=make_target(project,profile,a.cpu,result['run_id'],source if profile['units'] else None, validated if profile['units'] else None, Path(a.assets) if use_assets else None,assets if use_assets else []) row['target']=description evidence=out/ident;evidence.mkdir() for file in ('CastleEngineManifest.xml','target.json'): shutil.copyfile(project/file,evidence/file) shutil.copyfile(project/'code/ProbeImports.pas',evidence/'ProbeImports.pas') row['compile_link']=run_command(cge_command(a.cpu,'compile'),project,evidence/'compile-link.log',a.timeout) text=(evidence/'compile-link.log').read_text(errors='replace') if row['compile_link']['status']!='PASS': row['errors']=first_errors(text) or [text[-1500:]];row['triage']=classify(text);continue name='lib'+description['name']+'.so' outputs=list(project.rglob('lib'+description['name']+'*.so')) if not outputs:raise ProbeError('Compiler exited 0 but no expected native library matching: '+name) hashes={sha256(p) for p in outputs} if len(hashes)!=1:raise ProbeError('Ambiguous different native outputs') library=outputs[0];name=library.name;elf=inspect_elf(library.read_bytes(),a.cpu) row['native_library']={'file':str(library.relative_to(ROOT)),'sha256':sha256(library),'elf':elf} ppu_files={p.stem.casefold():p for p in project.rglob('*.ppu')} evidence_units=[] for unit in description['units']: p=ppu_files.get(unit.casefold()) if p is None or p.read_bytes()[:3]!=b'PPU': raise ProbeError('No fresh Pascal unit artifact found for '+unit) evidence_units.append({'unit':unit,'ppu':str(p.relative_to(ROOT)),'sha256':sha256(p), 'source_candidates':audit['units'].get(unit.casefold(),[])}) row['compiled_roots']=evidence_units if not pack:continue row['package']=run_command(cge_command(a.cpu,'package'),project,evidence/'package.log',a.timeout) if row['package']['status']!='PASS': row['errors']+=first_errors((evidence/'package.log').read_text(errors='replace'));continue candidates=list(project.rglob('*.apk')) if not candidates:raise ProbeError('Packaging exited 0 but no new APK') apk_hashes={sha256(p) for p in candidates} if len(apk_hashes)!=1:raise ProbeError('Multiple different APK outputs; refusing arbitrary selection') apk=candidates[0] apk_info=inspect_apk(apk,a.cpu,name,row['native_library']['sha256'],library.read_bytes()) signer=signature_tool() if signer: sign=run_command([signer,'verify','--verbose',str(apk)],project,evidence/'signature.log',120) row['signature']=sign if sign['status']!='PASS':row['package']['status']='FAIL';continue else:row['signature']={'status':'NOT_RUN','reason':'apksigner not found'} final=dist/(description['name']+'-'+a.cpu+'-debug.apk');shutil.copy2(apk,final) row['apk']={'file':str(final.relative_to(ROOT)),**apk_info} package_index.append({'profile':ident,'package':description['package'],'file':str(final.relative_to(ROOT)), 'sha256':sha256(final),'cpu':a.cpu,'build_id':result['run_id'], 'native_library_path_in_apk':apk_info['main_library_path']}) except (ProbeError,OSError,ValueError,KeyError,zipfile.BadZipFile) as e: row['errors'].append(str(e)) # A supposedly successful compiler/packager without valid output is a real test failure. if row['package']['status']=='PASS':row['package']['status']='FAIL' elif row['compile_link']['status']=='PASS':row['compile_link']['status']='FAIL' else:row['compile_link']={'status':'BLOCKED','reason':str(e)} finally: write_json(out/'result-in-progress.json',result) if need_source: try:verify(source) except ProbeError as e: result['source_modified_during_build']=str(e) for r in result['profiles']: if r['id']!='platform':r['compile_link']['status']='FAIL';r['errors'].append(str(e)) write_json(dist/'apks.json',{'build_id':result['run_id'],'packages':package_index}) write_json(ROOT/'dist/LATEST.json',{'build_id':result['run_id'],'directory':str(dist.relative_to(ROOT)), 'packages':package_index}) return finalize(result,out) def docker_build(a:argparse.Namespace,profiles:list[dict])->int: result,out=new_result(a.cpu,profiles,'Docker bootstrap not completed') if not shutil.which('docker'): for r in result['profiles']:r['compile_link']['reason']='Docker not installed. Use --local only with a configured CGE/FPC Android toolchain.' return finalize(result,out) try: lock=ROOT/'config/toolchain.lock.json' if lock.is_file(): image=json.loads(lock.read_text())['image'] if not re.fullmatch(r'[\w./-]+@sha256:[a-f0-9]{64}',image):raise ProbeError('Invalid toolchain digest lock') p=subprocess.run(['docker','image','inspect',image],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL) if p.returncode: r=run_command(['docker','pull','--platform=linux/amd64',image],ROOT,out/'docker-pull.log',a.timeout) if r['status']!='PASS':raise ProbeError('Cannot obtain locked Docker image; see docker-pull.log') else: r=run_command(['docker','pull','--platform=linux/amd64',IMAGE_TAG],ROOT,out/'docker-pull.log',a.timeout) if r['status']!='PASS':raise ProbeError('Docker image pull failed; see docker-pull.log') raw=subprocess.check_output(['docker','image','inspect',IMAGE_TAG],text=True) info=json.loads(raw)[0];digests=info.get('RepoDigests',[]) image=next((d for d in digests if 'castle-engine-cloud-builds-tools@sha256:' in d),None) if image is None:raise ProbeError('Image has no immutable RepoDigest') write_json(lock,{'image':image,'image_id':info['Id'],'resolved_from':IMAGE_TAG,'locked_at':stamp(), 'note':'Subsequent runs use this digest, never silently advance the tag.'}) cache=ROOT/'.cache/home';cache.mkdir(parents=True,exist_ok=True) cmd=['docker','run','--rm','--platform=linux/amd64','--user','0:0', '-e','HOME=/work/.cache/home','-v',str(ROOT)+':/work','-w','/work'] if os.environ.get('GITHUB_TOKEN'):cmd+=['-e','GITHUB_TOKEN'] if a.assets: resource=Path(a.assets).resolve() if not resource.is_dir():raise ProbeError('Asset directory not found: '+str(resource)) cmd+=['-v',str(resource)+':/probe-assets:ro'] cmd+=['--entrypoint','python3',image,'tools/runner.py','--local','--cpu',a.cpu, '--profiles',a.profiles,'--timeout',str(a.timeout)] if a.no_fetch:cmd+=['--no-fetch'] if a.package_all:cmd+=['--package-all'] if a.assets:cmd+=['--assets','/probe-assets'] for rel in a.asset_path or []:cmd+=['--asset-path',rel] write_json(out/'docker-command.json',{'command':cmd,'image':image}) # Stream the user's real build directly. No background processes/promises. latest=ROOT/'reports/LATEST.json' before=latest.read_bytes() if latest.exists() else None code=subprocess.call(cmd,cwd=ROOT) after=latest.read_bytes() if latest.exists() else None if code in (0,1,2) and after is not None and after!=before: fresh=json.loads(after) if (ROOT/fresh['directory']/'result.json').is_file():return code raise ProbeError('Container could not finish the report; Docker exit code '+str(code)) except (ProbeError,OSError,ValueError,KeyError,subprocess.CalledProcessError) as e: for r in result['profiles']:r['compile_link']['reason']=str(e) return finalize(result,out) def main()->int: ap=argparse.ArgumentParser(description=__doc__) ap.add_argument('--local',action='store_true',help='Use installed toolchain; still compile Android only') ap.add_argument('--cpu',choices=CPUS,default='aarch64') ap.add_argument('--profiles',default='all',help='all or comma-separated IDs from config/profiles.json') ap.add_argument('--no-fetch',action='store_true',help='Require previously fetched/verified source tree') ap.add_argument('--assets',help='Optional private KaM resource root; no resources are downloaded') ap.add_argument('--asset-path',action='append',help='Relative path under --assets to stage. Repeat for a small selected fixture set.') ap.add_argument('--package-all',action='store_true',help='Package every successfully linked probe for device startup checks') ap.add_argument('--timeout',type=int,default=1800,help='Per-stage safety timeout in seconds') a=ap.parse_args() if a.asset_path and not a.assets:ap.error('--asset-path requires --assets') if a.timeout<1:ap.error('--timeout must be positive') try:profiles=selected_profiles(a.profiles) except ProbeError as e:ap.error(str(e)) return build_local(a,profiles) if a.local else docker_build(a,profiles) if __name__=='__main__':raise SystemExit(main())