"""Fetch only source-text blobs at r16020. Never clone or download release archives. On the server, downloads Pascal/includes and dependency source, NOT data/maps/sprites, DLLs, .so, .a, .o, compiled resources, executables, or history. Requires Python stdlib. """ from __future__ import annotations import argparse, concurrent.futures, hashlib, json, os, re, time, urllib.error, urllib.parse, urllib.request from pathlib import Path, PurePosixPath from common import ROOT, COMMIT, REPOSITORY, ProbeError, git_blob, safe_relative, write_json, sha256, stamp CODE = {'.pas', '.pp', '.inc', '.inl', '.h', '.hpp', '.c', '.cpp', '.s', '.asm'} SKIP_PARTS = {'examples','example','demos','demo','tests','test','docs','doc','.git'} MAX_BLOB = 16 * 1024 * 1024 SUBMODULES = {'src/ext/pascalscript_submodule': 'reyandme/pascalscript'} def selected(path: str, main: bool = True) -> bool: safe_relative(path) p = PurePosixPath(path) if any(x.casefold() in SKIP_PARTS for x in p.parts[:-1]): return False license_name = p.name.casefold().startswith(('license','licence','copying','copyright')) if license_name: return p.suffix.casefold() in ('', '.txt', '.md', '.rst') if p.suffix.casefold() not in CODE: return False return not main or path.startswith('src/') or '/' not in path def request(url: str, max_bytes: int = 32*1024*1024) -> bytes: headers = {'User-Agent': 'KaM-Android-Architecture-Probe/0.3', 'Accept':'application/vnd.github+json'} # Never send a token to the raw-content host, never print tokens. token = os.environ.get('GITHUB_TOKEN') if token and urllib.parse.urlparse(url).hostname == 'api.github.com': headers['Authorization'] = 'Bearer ' + token error = None for attempt in range(3): try: with urllib.request.urlopen(urllib.request.Request(url, headers=headers), timeout=60) as r: result = r.read(max_bytes+1) if len(result) > max_bytes: raise ProbeError('Oversized source response: '+url) return result except (urllib.error.URLError, TimeoutError) as e: error = e if isinstance(e, urllib.error.HTTPError) and e.code in (401,403,404): break time.sleep(attempt+1) raise ProbeError('Source request failed: '+url+'; '+str(error)) def remote_tree(repo: str, ref: str) -> dict: if not re.fullmatch(r'[a-f0-9]{40}', ref): raise ProbeError('A full pinned commit is required') result = json.loads(request(f'https://api.github.com/repos/{repo}/git/trees/{ref}?recursive=1')) if result.get('truncated'): raise ProbeError('GitHub returned truncated tree; refusing incomplete source set') if not isinstance(result.get('tree'), list): raise ProbeError('Missing git tree') return result def enumerate_blobs(tree: dict, repo: str, ref: str, prefix: str = '', main: bool = True) -> list[dict]: result = [] for entry in tree['tree']: name = safe_relative(entry['path']) if entry['type'] != 'blob' or entry.get('mode') not in ('100644','100755'): continue if not selected(name, main): continue if not isinstance(entry.get('size'), int) or not 0 <= entry['size'] <= MAX_BLOB: raise ProbeError('Unsupported source size: '+name) result.append({'path':prefix+name, 'remote_path':name, 'repository':repo, 'commit':ref, 'git_blob_sha1':entry['sha'], 'size':entry['size']}) return result def fetch(destination: Path) -> dict: destination.mkdir(parents=True, exist_ok=True) tree = remote_tree(REPOSITORY, COMMIT) write_json(destination/'upstream-tree.json', tree) entries = enumerate_blobs(tree, REPOSITORY, COMMIT) modules=[] for item in tree['tree']: if item['type'] != 'commit' or not item['path'].startswith('src/'): continue name=safe_relative(item['path']); repo=SUBMODULES.get(name) if not repo: raise ProbeError('Unconfigured source submodule: '+name) child=remote_tree(repo,item['sha']) modules.append({'path':name,'repository':repo,'commit':item['sha']}) entries.extend(enumerate_blobs(child,repo,item['sha'],name+'/',False)) names=[e['path'] for e in entries] if len(names)!=len(set(names)): raise ProbeError('Overlapping source paths') if not entries: raise ProbeError('No source blobs selected') def obtain(e: dict) -> dict: target=destination/e['path'];target.parent.mkdir(parents=True,exist_ok=True) if target.is_symlink(): raise ProbeError('Symlink in source tree: '+str(target)) if target.exists(): data=target.read_bytes() if git_blob(data)!=e['git_blob_sha1']: raise ProbeError('Modified or corrupt baseline: '+str(target)+'; move it aside before refetch') else: rawpath=urllib.parse.quote(e['remote_path'],safe='/') data=request(f"https://raw.githubusercontent.com/{e['repository']}/{e['commit']}/{rawpath}",MAX_BLOB) if b'\0' in data: raise ProbeError('Non-text source blob rejected: '+e['path']) if len(data)!=e['size'] or git_blob(data)!=e['git_blob_sha1']: raise ProbeError('Source identity mismatch: '+e['path']) temp=target.with_name(target.name+'.download');temp.write_bytes(data);os.replace(temp,target) return {**e,'sha256':hashlib.sha256(data).hexdigest()} done=[] with concurrent.futures.ThreadPoolExecutor(max_workers=6) as pool: for e in pool.map(obtain,entries): done.append(e) if len(done)%100==0: print(f'Source only: {len(done)}/{len(entries)} files verified',flush=True) receipt={'schema':'kam-source-receipt/1','commit':COMMIT,'repository':REPOSITORY, 'downloaded_at':stamp(),'source_only':True,'submodules':modules, 'files':sorted(done,key=lambda e:e['path'])} write_json(destination/'SOURCE_RECEIPT.json',receipt) verify(destination) return receipt def verify(destination: Path) -> dict: file=destination/'SOURCE_RECEIPT.json' if not file.is_file(): raise ProbeError('Source receipt missing; run tools/source_fetch.py') d=json.loads(file.read_text()) if d.get('commit')!=COMMIT or d.get('repository')!=REPOSITORY or not d.get('source_only'): raise ProbeError('Wrong source baseline') expected=set() for item in d.get('files',[]): name=safe_relative(item['path']);p=destination/name if name in expected: raise ProbeError('Duplicate source receipt entry') expected.add(name) if p.is_symlink() or not p.is_file(): raise ProbeError('Source missing/linked: '+name) if p.stat().st_size!=item['size'] or sha256(p)!=item['sha256'] or git_blob(p.read_bytes())!=item['git_blob_sha1']: raise ProbeError('Source modified: '+name) if not expected: raise ProbeError('Empty source set') for p in destination.rglob('*'): if p.is_symlink(): raise ProbeError('Symlink in source baseline: '+str(p)) if p.is_file() and p.suffix.lower() in (CODE|{'.ppu','.o','.a','.so','.dll','.res'}): if p.relative_to(destination).as_posix() not in expected: raise ProbeError('Unrecorded source/binary in baseline: '+str(p)) return d def import_local(source: Path, destination: Path) -> dict: """Import an ALREADY pinned code-only receipt tree; no arbitrary git tree trusted.""" import shutil d=verify(source) if destination.exists() and any(destination.iterdir()): raise ProbeError('Destination must be empty') destination.mkdir(parents=True,exist_ok=True) for item in d['files']: p=destination/item['path'];p.parent.mkdir(parents=True,exist_ok=True) shutil.copy2(source/item['path'],p) write_json(destination/'SOURCE_RECEIPT.json',d) return verify(destination) if __name__=='__main__': ap=argparse.ArgumentParser(description=__doc__) ap.add_argument('--verify',action='store_true');ap.add_argument('--destination',type=Path,default=ROOT/'vendor/kam_remake') a=ap.parse_args() try: d=verify(a.destination) if a.verify else fetch(a.destination) print(f"Verified {len(d['files'])} source-text files, commit {COMMIT}") except (ProbeError,OSError,ValueError,KeyError) as e: print('SOURCE BLOCKED:',e);raise SystemExit(2)