Initial import
This commit is contained in:
admin_d committed
commit
dfcc43c72d
34 files changed
+3057
No files matched your search
Executable
+307
@@ -0,0 +1,307 @@
|
||||
"""One-command Android-only compile, full native link, APK and evidence pipeline."""
|
||||
from __future__ import annotations
|
||||
import argparse, json, os, re, shutil, subprocess, sys, uuid, zipfile
|
||||
from pathlib import Path
|
||||
from common import ROOT, COMMIT, CPUS, ProbeError, stamp, sha256, write_json, run_command, first_errors, classify
|
||||
from source_fetch import fetch, verify
|
||||
from source_audit import inventory
|
||||
from generate import make_target, asset_inventory
|
||||
from artifacts import inspect_elf, inspect_apk
|
||||
|
||||
IMAGE_TAG='kambi/castle-engine-cloud-builds-tools:cge-unstable'
|
||||
PROFILES=json.loads((ROOT/'config/profiles.json').read_text())
|
||||
|
||||
def selected_profiles(text:str)->list[dict]:
|
||||
if text=='all':return PROFILES
|
||||
names=[s.strip() for s in text.split(',') if s.strip()]
|
||||
available={p['id'] for p in PROFILES}
|
||||
if not names or set(names)-available:raise ProbeError('Unknown/empty profiles: '+text)
|
||||
names=set(names)|{'platform'} # Always establish the same-toolchain platform baseline.
|
||||
return [p for p in PROFILES if p['id'] in names]
|
||||
|
||||
def cge_command(cpu:str,action:str)->list[str]:
|
||||
if cpu not in CPUS:raise ProbeError('Unsupported Android CPU')
|
||||
if action not in ('compile','package'):raise ProbeError('Not an Android build action')
|
||||
cmd=['castle-engine','--os=android','--cpu='+cpu,'--compiler=fpc','--mode=debug']
|
||||
if action=='package':cmd+=['--assume-compiled','--package-format=android-apk']
|
||||
return cmd+[action]
|
||||
|
||||
def blocked_row(profile:dict,reason:str)->dict:
|
||||
return {'id':profile['id'],'required':profile['required'],'purpose':profile['purpose'],
|
||||
'compile_link':{'status':'BLOCKED','reason':reason},'package':{'status':'NOT_RUN'},
|
||||
'runtime':{'status':'NOT_RUN'},'errors':[]}
|
||||
|
||||
def gate(rows:list[dict])->str:
|
||||
by_id={r['id']:r for r in rows}
|
||||
required=[p for p in PROFILES if p['required']]
|
||||
if any(r['compile_link']['status']=='FAIL' or r['package']['status']=='FAIL' for r in rows if r['required']):
|
||||
return 'NOT_BUILDABLE_AS_IS'
|
||||
if any(r['compile_link']['status']=='BLOCKED' for r in rows if r['required']):return 'BLOCKED'
|
||||
if any(p['id'] not in by_id for p in required):return 'PARTIAL_SELECTION'
|
||||
for p in required:
|
||||
r=by_id[p['id']]
|
||||
if r['compile_link']['status']!='PASS':return 'BLOCKED'
|
||||
if p['package'] and r['package']['status']!='PASS':return 'BLOCKED'
|
||||
return 'ANDROID_BUILD_GATE_PASS'
|
||||
|
||||
def render_summary(result:dict,out:Path)->None:
|
||||
title='# KaM r16020 — Android architecture probe\n\n'
|
||||
title+=f"Build ID: `{result['run_id']}` · CPU: `{result['cpu']}` · Gate: **{result['gate']}**\n\n"
|
||||
title+='Нет desktop target. Успех означает сборку выбранных Android-корней, не игровую совместимость.\n\n'
|
||||
title+='| Проба | Compile + native link | APK | Ошибка / следующий шаг |\n|---|---|---|---|\n'
|
||||
for r in result['profiles']:
|
||||
error=(r.get('errors') or [r['compile_link'].get('reason','')])[0]
|
||||
error=error.replace('|','/').replace('\n',' ')[:260]
|
||||
title+=f"| {r['id']} | {r['compile_link']['status']} | {r['package']['status']} | {error} |\n"
|
||||
title+='\n## Что установлено\n\n'
|
||||
title+='`PASS` compile_link = FPC завершился успешно, найден свежий ELF нужного ABI, проверены корневые PPU. Линковка не отключена; требуются определённые символы. '
|
||||
title+='`PASS` APK = упаковка завершилась, проверены ZIP/DEX, правильный ABI, совпадение native library и DT_NEEDED. '
|
||||
title+='Подпись проверяется отдельно, если доступен apksigner. `runtime=NOT_RUN` до реального запуска на Android.\n\n'
|
||||
title+='`NOT_BUILDABLE_AS_IS` — найдено препятствие в текущем коде/окружении, а не доказательство, что core надо переписать. '
|
||||
title+='Ошибки legacy_render/legacy_audio диагностические и не входят в обязательный gate, но их транзитивное попадание в core всё равно видно.\n\n'
|
||||
title+='## Как принимать решение\n\n'
|
||||
title+='| Свидетельство | Разумное решение |\n|---|---|\n'
|
||||
title+='| Корень собирается с оригинальными исходниками | Кандидат на reuse; поведение ещё проверяется отдельно |\n'
|
||||
title+='| Активная зависимость Windows/LCL/desktop GL | Исследовать минимальное отделение платформенной части |\n'
|
||||
title+='| Отсутствует source unit / package | Проверить комплектность исходников, условные defines и пакет для Android |\n'
|
||||
title+='| Undefined reference / отсутствует native library | Сборка/подключение Android-библиотеки либо адаптер Castle |\n'
|
||||
title+='| Один и тот же блокер у многих модулей | Исправить общий корень и повторить пробы; не считать это множеством отдельных переписываний |\n'
|
||||
title+='| Срабатывает exception при запуске | Снять logcat; проверить unit initialization/lifecycle/доступ к данным |\n'
|
||||
title+='\nНи один автоматический результат здесь не выносит решение «написать подсистему с нуля».\n'
|
||||
(out/'SUMMARY.md').write_text(title,encoding='utf-8')
|
||||
|
||||
def finalize(result:dict,out:Path)->int:
|
||||
result['gate']=gate(result['profiles'])
|
||||
write_json(out/'result.json',result);render_summary(result,out)
|
||||
write_json(ROOT/'reports/LATEST.json',{'run_id':result['run_id'],'directory':str(out.relative_to(ROOT)),
|
||||
'gate':result['gate'],'summary':str((out/'SUMMARY.md').relative_to(ROOT))})
|
||||
shutil.copyfile(out/'SUMMARY.md',ROOT/'reports/LATEST-SUMMARY.md')
|
||||
# Evidence intentionally excludes APK/.so, user game assets, caches and source trees.
|
||||
with zipfile.ZipFile(out/'evidence.zip','w',zipfile.ZIP_DEFLATED) as z:
|
||||
for p in sorted(out.rglob('*')):
|
||||
if p.is_file() and p.suffix in ('.json','.log','.md','.txt','.xml','.pas'):
|
||||
z.write(p,p.relative_to(out))
|
||||
print('\n'+result['gate'])
|
||||
print('Report: '+str(out/'SUMMARY.md'))
|
||||
print('Evidence (no game assets): '+str(out/'evidence.zip'))
|
||||
return 0 if result['gate']=='ANDROID_BUILD_GATE_PASS' else 1 if result['gate']=='NOT_BUILDABLE_AS_IS' else 2
|
||||
|
||||
def new_result(cpu:str,profiles:list[dict],reason:str='Not started')->tuple[dict,Path]:
|
||||
rid=stamp()+'-'+uuid.uuid4().hex[:8]
|
||||
out=ROOT/'reports/runs'/rid;out.mkdir(parents=True)
|
||||
return ({'schema':'kam-android-build/1','run_id':rid,'cpu':cpu,'commit':COMMIT,
|
||||
'desktop_target':False,'shims':False,'runtime':'NOT_RUN',
|
||||
'profiles':[blocked_row(p,reason) for p in profiles]},out)
|
||||
|
||||
def extra_paths()->dict:
|
||||
d=json.loads((ROOT/'config/extra-paths.json').read_text())
|
||||
for key in ('source_paths','library_paths'):
|
||||
if not isinstance(d.get(key),list) or not all(isinstance(s,str) for s in d[key]):
|
||||
raise ProbeError('extra-paths.json must contain string arrays')
|
||||
d[key]=[str((ROOT/p).resolve()) if not Path(p).is_absolute() else str(Path(p).resolve()) for p in d[key]]
|
||||
return d
|
||||
|
||||
def tool_versions(out:Path)->dict:
|
||||
commands={'castle':['castle-engine','--version'],'fpc':['fpc','-iV'],'java':['java','-version'],
|
||||
'adb':['adb','version']}
|
||||
d={}
|
||||
for name,cmd in commands.items():
|
||||
if shutil.which(cmd[0]):
|
||||
rec=run_command(cmd,ROOT,out/(name+'-version.log'),60)
|
||||
rec['output']=(out/(name+'-version.log')).read_text(errors='replace')[:16000];d[name]=rec
|
||||
else:d[name]={'status':'NOT_INSTALLED'}
|
||||
d['paths']={k:os.environ[k] for k in ('CASTLE_ENGINE_PATH','ANDROID_HOME','ANDROID_SDK_ROOT',
|
||||
'ANDROID_NDK_ROOT','JAVA_HOME','FPCDIR') if k in os.environ}
|
||||
for variable in ('ANDROID_NDK_ROOT','ANDROID_HOME','ANDROID_SDK_ROOT'):
|
||||
if os.environ.get(variable):
|
||||
p=Path(os.environ[variable])/'source.properties'
|
||||
if p.is_file():d[variable+'_source_properties']=p.read_text(errors='replace')
|
||||
lock=ROOT/'config/toolchain.lock.json'
|
||||
if lock.is_file():d['docker_image_lock']=json.loads(lock.read_text())
|
||||
write_json(out/'toolchain.json',d)
|
||||
return d
|
||||
|
||||
def signature_tool()->str|None:
|
||||
direct=shutil.which('apksigner')
|
||||
if direct:return direct
|
||||
paths=[]
|
||||
for key in ('ANDROID_HOME','ANDROID_SDK_ROOT'):
|
||||
if os.environ.get(key):paths+=list(Path(os.environ[key]).glob('build-tools/*/apksigner'))
|
||||
return str(sorted(paths)[-1]) if paths else None
|
||||
|
||||
def build_local(a:argparse.Namespace,profiles:list[dict])->int:
|
||||
result,out=new_result(a.cpu,profiles)
|
||||
result['toolchain']=tool_versions(out)
|
||||
if not shutil.which('castle-engine'):
|
||||
for r in result['profiles']:r['compile_link']['reason']='castle-engine not installed; no Android compilation attempted'
|
||||
return finalize(result,out)
|
||||
source=ROOT/'vendor/kam_remake';audit=None;source_error='';assets=[]
|
||||
need_source=any(p['units'] for p in profiles)
|
||||
try:
|
||||
extras=extra_paths()
|
||||
assets=asset_inventory(Path(a.assets),a.asset_path) if a.assets else []
|
||||
result['asset_count']=len(assets)
|
||||
if need_source:
|
||||
if (source/'SOURCE_RECEIPT.json').is_file():receipt=verify(source)
|
||||
elif a.no_fetch:raise ProbeError('Pinned source tree absent and --no-fetch requested')
|
||||
else:receipt=fetch(source)
|
||||
write_json(out/'source-receipt.json',receipt)
|
||||
audit=inventory(source);write_json(out/'source-inventory.json',audit)
|
||||
for path in extras['source_paths']:
|
||||
for p in Path(path).rglob('*.pas'):
|
||||
if p.stem.casefold() in audit['units']:
|
||||
raise ProbeError('Extra path shadows original unit '+p.stem+'; no shims allowed')
|
||||
except (ProbeError,OSError,ValueError,KeyError) as e:
|
||||
source_error=str(e);extras={'source_paths':[],'library_paths':[]}
|
||||
result['source_or_inputs_error']=source_error
|
||||
work=ROOT/'build'/result['run_id'];work.mkdir(parents=True)
|
||||
dist=ROOT/'dist'/result['run_id'];dist.mkdir(parents=True)
|
||||
package_index=[]
|
||||
for idx,profile in enumerate(profiles):
|
||||
row=result['profiles'][idx];ident=profile['id'];project=work/ident
|
||||
print(f"[{idx+1}/{len(profiles)}] Android {a.cpu}: {ident}",flush=True)
|
||||
try:
|
||||
if profile['units']:
|
||||
if source_error:raise ProbeError(source_error)
|
||||
missing=[u for u in profile['units'] if u.casefold() not in audit['units']]
|
||||
if missing:raise ProbeError('Missing required original source units: '+', '.join(missing))
|
||||
# Resources only enter APKs when explicitly requested. The platform baseline remains independent.
|
||||
pack=profile['package'] or a.package_all
|
||||
use_assets=bool(a.assets) and pack and ident=='platform'
|
||||
description=make_target(project,profile,a.cpu,result['run_id'],source if profile['units'] else None,
|
||||
extras if profile['units'] else {'source_paths':[],'library_paths':[]},
|
||||
Path(a.assets) if use_assets else None,assets if use_assets else [])
|
||||
row['target']=description
|
||||
evidence=out/ident;evidence.mkdir()
|
||||
for file in ('CastleEngineManifest.xml','target.json'):
|
||||
shutil.copyfile(project/file,evidence/file)
|
||||
shutil.copyfile(project/'code/ProbeImports.pas',evidence/'ProbeImports.pas')
|
||||
row['compile_link']=run_command(cge_command(a.cpu,'compile'),project,evidence/'compile-link.log',a.timeout)
|
||||
text=(evidence/'compile-link.log').read_text(errors='replace')
|
||||
if row['compile_link']['status']!='PASS':
|
||||
row['errors']=first_errors(text) or [text[-1500:]];row['triage']=classify(text);continue
|
||||
name='lib'+description['name']+'.so'
|
||||
outputs=list(project.rglob('lib'+description['name']+'*.so'))
|
||||
if not outputs:raise ProbeError('Compiler exited 0 but no expected native library matching: '+name)
|
||||
hashes={sha256(p) for p in outputs}
|
||||
if len(hashes)!=1:raise ProbeError('Ambiguous different native outputs')
|
||||
library=outputs[0];name=library.name;elf=inspect_elf(library.read_bytes(),a.cpu)
|
||||
row['native_library']={'file':str(library.relative_to(ROOT)),'sha256':sha256(library),'elf':elf}
|
||||
ppu_files={p.stem.casefold():p for p in project.rglob('*.ppu')}
|
||||
evidence_units=[]
|
||||
for unit in description['units']:
|
||||
p=ppu_files.get(unit.casefold())
|
||||
if p is None or p.read_bytes()[:3]!=b'PPU':
|
||||
raise ProbeError('No fresh Pascal unit artifact found for '+unit)
|
||||
evidence_units.append({'unit':unit,'ppu':str(p.relative_to(ROOT)),'sha256':sha256(p),
|
||||
'source_candidates':audit['units'].get(unit.casefold(),[])})
|
||||
row['compiled_roots']=evidence_units
|
||||
if not pack:continue
|
||||
row['package']=run_command(cge_command(a.cpu,'package'),project,evidence/'package.log',a.timeout)
|
||||
if row['package']['status']!='PASS':
|
||||
row['errors']+=first_errors((evidence/'package.log').read_text(errors='replace'));continue
|
||||
candidates=list(project.rglob('*.apk'))
|
||||
if not candidates:raise ProbeError('Packaging exited 0 but no new APK')
|
||||
apk_hashes={sha256(p) for p in candidates}
|
||||
if len(apk_hashes)!=1:raise ProbeError('Multiple different APK outputs; refusing arbitrary selection')
|
||||
apk=candidates[0]
|
||||
apk_info=inspect_apk(apk,a.cpu,name,row['native_library']['sha256'],library.read_bytes())
|
||||
signer=signature_tool()
|
||||
if signer:
|
||||
sign=run_command([signer,'verify','--verbose',str(apk)],project,evidence/'signature.log',120)
|
||||
row['signature']=sign
|
||||
if sign['status']!='PASS':row['package']['status']='FAIL';continue
|
||||
else:row['signature']={'status':'NOT_RUN','reason':'apksigner not found'}
|
||||
final=dist/(description['name']+'-'+a.cpu+'-debug.apk');shutil.copy2(apk,final)
|
||||
row['apk']={'file':str(final.relative_to(ROOT)),**apk_info}
|
||||
package_index.append({'profile':ident,'package':description['package'],'file':str(final.relative_to(ROOT)),
|
||||
'sha256':sha256(final),'cpu':a.cpu,'build_id':result['run_id'],
|
||||
'native_library_path_in_apk':apk_info['main_library_path']})
|
||||
except (ProbeError,OSError,ValueError,KeyError,zipfile.BadZipFile) as e:
|
||||
row['errors'].append(str(e))
|
||||
# A supposedly successful compiler/packager without valid output is a real test failure.
|
||||
if row['package']['status']=='PASS':row['package']['status']='FAIL'
|
||||
elif row['compile_link']['status']=='PASS':row['compile_link']['status']='FAIL'
|
||||
else:row['compile_link']={'status':'BLOCKED','reason':str(e)}
|
||||
finally:
|
||||
write_json(out/'result-in-progress.json',result)
|
||||
if need_source and not source_error:
|
||||
try:verify(source)
|
||||
except ProbeError as e:
|
||||
result['source_modified_during_build']=str(e)
|
||||
for r in result['profiles']:
|
||||
if r['id']!='platform':r['compile_link']['status']='FAIL';r['errors'].append(str(e))
|
||||
write_json(dist/'apks.json',{'build_id':result['run_id'],'packages':package_index})
|
||||
write_json(ROOT/'dist/LATEST.json',{'build_id':result['run_id'],'directory':str(dist.relative_to(ROOT)),
|
||||
'packages':package_index})
|
||||
return finalize(result,out)
|
||||
|
||||
def docker_build(a:argparse.Namespace,profiles:list[dict])->int:
|
||||
result,out=new_result(a.cpu,profiles,'Docker bootstrap not completed')
|
||||
if not shutil.which('docker'):
|
||||
for r in result['profiles']:r['compile_link']['reason']='Docker not installed. Use --local only with a configured CGE/FPC Android toolchain.'
|
||||
return finalize(result,out)
|
||||
try:
|
||||
lock=ROOT/'config/toolchain.lock.json'
|
||||
if lock.is_file():
|
||||
image=json.loads(lock.read_text())['image']
|
||||
if not re.fullmatch(r'[\w./-]+@sha256:[a-f0-9]{64}',image):raise ProbeError('Invalid toolchain digest lock')
|
||||
p=subprocess.run(['docker','image','inspect',image],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)
|
||||
if p.returncode:
|
||||
r=run_command(['docker','pull','--platform=linux/amd64',image],ROOT,out/'docker-pull.log',a.timeout)
|
||||
if r['status']!='PASS':raise ProbeError('Cannot obtain locked Docker image; see docker-pull.log')
|
||||
else:
|
||||
r=run_command(['docker','pull','--platform=linux/amd64',IMAGE_TAG],ROOT,out/'docker-pull.log',a.timeout)
|
||||
if r['status']!='PASS':raise ProbeError('Docker image pull failed; see docker-pull.log')
|
||||
raw=subprocess.check_output(['docker','image','inspect',IMAGE_TAG],text=True)
|
||||
info=json.loads(raw)[0];digests=info.get('RepoDigests',[])
|
||||
image=next((d for d in digests if 'castle-engine-cloud-builds-tools@sha256:' in d),None)
|
||||
if image is None:raise ProbeError('Image has no immutable RepoDigest')
|
||||
write_json(lock,{'image':image,'image_id':info['Id'],'resolved_from':IMAGE_TAG,'locked_at':stamp(),
|
||||
'note':'Subsequent runs use this digest, never silently advance the tag.'})
|
||||
cache=ROOT/'.cache/home';cache.mkdir(parents=True,exist_ok=True)
|
||||
cmd=['docker','run','--rm','--platform=linux/amd64','--user','0:0',
|
||||
'-e','HOME=/work/.cache/home','-v',str(ROOT)+':/work','-w','/work']
|
||||
if os.environ.get('GITHUB_TOKEN'):cmd+=['-e','GITHUB_TOKEN']
|
||||
if a.assets:
|
||||
resource=Path(a.assets).resolve()
|
||||
if not resource.is_dir():raise ProbeError('Asset directory not found: '+str(resource))
|
||||
cmd+=['-v',str(resource)+':/probe-assets:ro']
|
||||
cmd+=['--entrypoint','python3',image,'tools/runner.py','--local','--cpu',a.cpu,
|
||||
'--profiles',a.profiles,'--timeout',str(a.timeout)]
|
||||
if a.no_fetch:cmd+=['--no-fetch']
|
||||
if a.package_all:cmd+=['--package-all']
|
||||
if a.assets:cmd+=['--assets','/probe-assets']
|
||||
for rel in a.asset_path or []:cmd+=['--asset-path',rel]
|
||||
write_json(out/'docker-command.json',{'command':cmd,'image':image})
|
||||
# Stream the user's real build directly. No background processes/promises.
|
||||
latest=ROOT/'reports/LATEST.json'
|
||||
before=latest.read_bytes() if latest.exists() else None
|
||||
code=subprocess.call(cmd,cwd=ROOT)
|
||||
after=latest.read_bytes() if latest.exists() else None
|
||||
if code in (0,1,2) and after is not None and after!=before:
|
||||
fresh=json.loads(after)
|
||||
if (ROOT/fresh['directory']/'result.json').is_file():return code
|
||||
raise ProbeError('Container could not finish the report; Docker exit code '+str(code))
|
||||
except (ProbeError,OSError,ValueError,KeyError,subprocess.CalledProcessError) as e:
|
||||
for r in result['profiles']:r['compile_link']['reason']=str(e)
|
||||
return finalize(result,out)
|
||||
|
||||
def main()->int:
|
||||
ap=argparse.ArgumentParser(description=__doc__)
|
||||
ap.add_argument('--local',action='store_true',help='Use installed toolchain; still compile Android only')
|
||||
ap.add_argument('--cpu',choices=CPUS,default='aarch64')
|
||||
ap.add_argument('--profiles',default='all',help='all or comma-separated IDs from config/profiles.json')
|
||||
ap.add_argument('--no-fetch',action='store_true',help='Require previously fetched/verified source tree')
|
||||
ap.add_argument('--assets',help='Optional private KaM resource root; no resources are downloaded')
|
||||
ap.add_argument('--asset-path',action='append',help='Relative path under --assets to stage. Repeat for a small selected fixture set.')
|
||||
ap.add_argument('--package-all',action='store_true',help='Package every successfully linked probe for device startup checks')
|
||||
ap.add_argument('--timeout',type=int,default=1800,help='Per-stage safety timeout in seconds')
|
||||
a=ap.parse_args()
|
||||
if a.asset_path and not a.assets:ap.error('--asset-path requires --assets')
|
||||
if a.timeout<1:ap.error('--timeout must be positive')
|
||||
try:profiles=selected_profiles(a.profiles)
|
||||
except ProbeError as e:ap.error(str(e))
|
||||
return build_local(a,profiles) if a.local else docker_build(a,profiles)
|
||||
|
||||
if __name__=='__main__':raise SystemExit(main())
|
||||
Reference in new issue
Block a user