Reuse Android compiler outputs and remove old build artifacts

tools/build_cache.py

@@ -0,0 +1,40 @@
+"""Keep compiler outputs while their source selection and toolchain stay compatible."""
+from __future__ import annotations
+import json, os, shutil
+from pathlib import Path
+from common import ROOT, COMMIT, CPUS, ProbeError, sha256, write_json
+
+def cached_project(cpu: str, resolution: dict, logs: Path) -> Path:
+    if cpu not in CPUS:raise ProbeError('Unsupported Android CPU')
+    cache=ROOT/'build/work'/cpu
+    for directory in (ROOT/'build',ROOT/'build/work',cache):
+        if directory.is_symlink():raise ProbeError('Build cache must not be a symlink: '+str(directory))
+    cache.mkdir(parents=True,exist_ok=True)
+    project=cache/'application'
+    if project.is_symlink():raise ProbeError('Cached project must not be a symlink')
+    inputs={
+        'schema':1,'cpu':cpu,'upstream_commit':COMMIT,
+        'manifest':sha256(ROOT/'app/CastleEngineManifest.xml'),
+        'source_contract':sha256(ROOT/'config/extra-paths.json'),
+        'paths':{key:[str(path) for path in resolution[key]]
+                 for key in ('compiler_source_paths','library_paths')},
+        'environment':{key:os.environ.get(key) for key in
+            ('KAM_TOOLCHAIN_IMAGE','CASTLE_ENGINE_PATH','FPCDIR','ANDROID_NDK_ROOT',
+             'ANDROID_HOME','ANDROID_SDK_ROOT','JAVA_HOME')},
+        # Removing or adding a unit can change compiler precedence even at the same paths.
+        'source_files':sorted({str(path) for directory in resolution['compiler_source_paths']
+            for path in directory.iterdir() if path.suffix.lower() in ('.pas','.pp','.inc') and path.is_file()}),
+    }
+    if not os.environ.get('KAM_TOOLCHAIN_IMAGE'):
+        inputs['tools']={}
+        for name in ('castle-engine','fpc'):
+            executable=shutil.which(name)
+            inputs['tools'][name]=None if executable is None else sha256(Path(executable).resolve())
+    marker=cache/'inputs.json'
+    try:previous=json.loads(marker.read_text())
+    except (FileNotFoundError,ValueError):previous=None
+    reused=project.is_dir() and previous==inputs
+    if project.exists() and not reused:shutil.rmtree(project)
+    write_json(marker,inputs)
+    write_json(logs/'compiler-cache.json',{'project':str(project),'reused':reused,'inputs':inputs})
+    return project

tools/generate.py

@@ -1,11 +1,11 @@
 """Generate isolated Android CGE targets. No KaM stubs or rewritten game loaders."""
 from __future__ import annotations
-import hashlib, os, re, shutil, xml.etree.ElementTree as ET
+import hashlib, io, os, re, shutil, xml.etree.ElementTree as ET
 from pathlib import Path
 from common import ROOT, ProbeError
 
 UPSTREAM_DEFINES=['DBG_SKIP_SECURE_AUTH']  # Existing upstream switch; private multiplayer authentication is out of scope.
-COMPILER_OPTIONS=['-B','-CX-','-XX-','-k--no-undefined','-k--no-gc-sections']
+COMPILER_OPTIONS=['-CX-','-XX-','-k--no-undefined','-k--no-gc-sections']
 _ANDROID_SERVICE_NAME=re.compile(r'[A-Za-z][A-Za-z0-9_]*\Z')
 
 _PROJECT_ATTRIBUTES={
@@ -89,27 +89,36 @@ def android_service_roots(root: Path) -> tuple[Path,...]:
         raise ProbeError(f'Cannot inspect Android services: {error}') from error
 
 def materialize_application_project(project: Path, source_resolution: dict) -> dict:
-    """Create one fresh Castle project from the permanent manifest and RP01 paths."""
-    if project.exists():raise ProbeError('Target directory must be fresh: '+str(project))
+    """Refresh project inputs while preserving compatible compiler outputs."""
+    if project.is_symlink():raise ProbeError('Target directory must not be a symlink: '+str(project))
     try:
         compiler_paths=tuple(source_resolution['compiler_source_paths'])
         library_paths=tuple(source_resolution['library_paths'])
     except (KeyError, TypeError) as error:
         raise ProbeError(f'Invalid source resolution: {error}') from error
     tree,root=_permanent_manifest()
-    project.mkdir(parents=True)
-    data_root=project/'data';data_root.mkdir()
+    project.mkdir(parents=True,exist_ok=True)
+    data_root=project/'data'
+    if data_root.is_symlink():raise ProbeError('Project data must not be a symlink')
+    data_root.mkdir(exist_ok=True)
     options=_one_child(root,'compiler_options')
     for tag,paths,kind in (('search_paths',compiler_paths,'Source'),('library_paths',library_paths,'Native library')):
         slot=_one_child(options,tag)
         for child in list(slot):slot.remove(child)
         for path in paths:ET.SubElement(slot,'path',value=_manifest_path(path,project,kind))
     ET.indent(tree)
-    tree.write(project/'CastleEngineManifest.xml',encoding='utf-8',xml_declaration=True)
+    rendered=io.BytesIO()
+    tree.write(rendered,encoding='utf-8',xml_declaration=True)
+    manifest=project/'CastleEngineManifest.xml'
+    if manifest.is_symlink():raise ProbeError('Generated manifest must not be a symlink')
+    if not manifest.exists() or manifest.read_bytes()!=rendered.getvalue():
+        manifest.write_bytes(rendered.getvalue())
     source_services=ROOT/'app/android_services'
     services=android_service_roots(source_services)
+    destination_root=project/'android_services'
+    android_service_roots(destination_root)
+    if destination_root.exists():shutil.rmtree(destination_root)
     if services:
-        destination_root=project/'android_services'
         destination_root.mkdir()
         for service in services:
             destination=destination_root/service.name

tools/runner.py

@@ -1,6 +1,6 @@
 """One-command Android-only compile, full native link, APK and evidence pipeline."""
 from __future__ import annotations
-import argparse, json, os, re, shutil, subprocess, sys, uuid, zipfile
+import argparse, fcntl, json, os, re, shutil, subprocess, sys, uuid, zipfile
 from pathlib import Path
 from common import (ROOT, COMMIT, CPUS, ProbeError, stamp, sha256, write_json, run_command,
                     first_errors, CURRENT_OUTPUT_DIR, CURRENT_BUILD_METADATA,
@@ -12,6 +12,7 @@ from source_resolution import load_source_contract, validate_source_contract, bu
 from generate import android_service_roots, materialize_application_project
 from artifacts import inspect_elf, inspect_apk
 from resource_staging import stage_runtime_resources
+from build_cache import cached_project
 
 IMAGE_TAG='kambi/castle-engine-cloud-builds-tools:cge-stable'
 BUILD_EVIDENCE_DIR=ROOT/'build-evidence'
@@ -152,9 +153,11 @@ def build_local(a:argparse.Namespace,build_id:str)->int:
         audited=inventory(vendor_root)
         validated=validate_source_contract(load_source_contract(),vendor_root,receipt,audited)
         resolution=build_source_resolution(validated,vendor_root,ROOT/'code')
-        descriptor=materialize_application_project(work/'application',resolution);app=_application(descriptor)
+        descriptor=materialize_application_project(cached_project(a.cpu,resolution,logs),resolution);app=_application(descriptor)
         resources=stage_runtime_resources(descriptor['data_root'])
         stage_android_services(descriptor['project_dir'],logs)
+        for pattern in (f"lib{descriptor['name']}_android_{a.cpu}.so",'*.apk'):
+            for output in descriptor['project_dir'].rglob(pattern):output.unlink()
     except (ProbeError,OSError,ValueError,KeyError) as error:
         return _finish(_record(build_id,a.cpu,'BLOCKED',app,None,compile,package,error=str(error)))
     if not shutil.which('castle-engine'):
@@ -219,7 +222,8 @@ def docker_build(a:argparse.Namespace,build_id:str)->int:
             if image is None:raise ProbeError('Image has no immutable RepoDigest')
             write_json(lock,{'image':image,'image_id':info['Id'],'resolved_from':IMAGE_TAG,'locked_at':stamp(),'note':'Subsequent runs use this digest, never silently advance the tag.'})
         cmd=['docker','run','--rm','--platform=linux/amd64','--user','0:0','-e','HOME=/work/.cache/home',
-             '-e','KAM_BUILD_ID='+build_id,'-e','KAM_CONTAINER_BUILD=1','-v',str(ROOT)+':/work','-w','/work']
+             '-e','KAM_BUILD_ID='+build_id,'-e','KAM_CONTAINER_BUILD=1',
+             '-e','KAM_TOOLCHAIN_IMAGE='+image,'-v',str(ROOT)+':/work','-w','/work']
         if os.environ.get('GITHUB_TOKEN'):cmd+=['-e','GITHUB_TOKEN']
         cmd+=['--entrypoint','python3',image,'tools/runner.py','--local','--cpu',a.cpu,'--timeout',str(a.timeout)]
         if a.no_fetch:cmd+=['--no-fetch']
@@ -242,7 +246,13 @@ def main()->int:
     inherited=os.environ.get('KAM_BUILD_ID')
     if inherited and not re.fullmatch(r'\d{8}T\d{6}Z-[0-9a-f]{8}',inherited):ap.error('Invalid KAM_BUILD_ID')
     build_id=inherited or stamp()+'-'+uuid.uuid4().hex[:8]
-    result=build_local(a,build_id) if a.local else docker_build(a,build_id)
+    if os.environ.get('KAM_CONTAINER_BUILD')=='1':
+        result=build_local(a,build_id) if a.local else docker_build(a,build_id)
+    else:
+        (ROOT/'build').mkdir(exist_ok=True)
+        with (ROOT/'build/runner.lock').open('a') as lock:
+            fcntl.flock(lock,fcntl.LOCK_EX)
+            result=build_local(a,build_id) if a.local else docker_build(a,build_id)
     if result==0 and os.environ.get('KAM_CONTAINER_BUILD')!='1':
         subprocess.run([sys.executable,str(ROOT/'tools/device.py')],cwd=ROOT,check=False)
     return result