Initial import

tools/source_fetch.py

@@ -0,0 +1,147 @@
+"""Fetch only source-text blobs at r16020. Never clone or download release archives.
+On the server, downloads Pascal/includes and dependency source, NOT data/maps/sprites,
+DLLs, .so, .a, .o, compiled resources, executables, or history. Requires Python stdlib.
+"""
+from __future__ import annotations
+import argparse, concurrent.futures, hashlib, json, os, re, time, urllib.error, urllib.parse, urllib.request
+from pathlib import Path, PurePosixPath
+from common import ROOT, COMMIT, REPOSITORY, ProbeError, git_blob, safe_relative, write_json, sha256, stamp
+
+CODE = {'.pas', '.pp', '.inc', '.inl', '.h', '.hpp', '.c', '.cpp', '.s', '.asm'}
+SKIP_PARTS = {'examples','example','demos','demo','tests','test','docs','doc','.git'}
+MAX_BLOB = 16 * 1024 * 1024
+SUBMODULES = {'src/ext/pascalscript_submodule': 'reyandme/pascalscript'}
+
+def selected(path: str, main: bool = True) -> bool:
+    safe_relative(path)
+    p = PurePosixPath(path)
+    if any(x.casefold() in SKIP_PARTS for x in p.parts[:-1]): return False
+    license_name = p.name.casefold().startswith(('license','licence','copying','copyright'))
+    if license_name: return p.suffix.casefold() in ('', '.txt', '.md', '.rst')
+    if p.suffix.casefold() not in CODE: return False
+    return not main or path.startswith('src/') or '/' not in path
+
+def request(url: str, max_bytes: int = 32*1024*1024) -> bytes:
+    headers = {'User-Agent': 'KaM-Android-Architecture-Probe/0.3', 'Accept':'application/vnd.github+json'}
+    # Never send a token to the raw-content host, never print tokens.
+    token = os.environ.get('GITHUB_TOKEN')
+    if token and urllib.parse.urlparse(url).hostname == 'api.github.com':
+        headers['Authorization'] = 'Bearer ' + token
+    error = None
+    for attempt in range(3):
+        try:
+            with urllib.request.urlopen(urllib.request.Request(url, headers=headers), timeout=60) as r:
+                result = r.read(max_bytes+1)
+            if len(result) > max_bytes: raise ProbeError('Oversized source response: '+url)
+            return result
+        except (urllib.error.URLError, TimeoutError) as e:
+            error = e
+            if isinstance(e, urllib.error.HTTPError) and e.code in (401,403,404): break
+            time.sleep(attempt+1)
+    raise ProbeError('Source request failed: '+url+'; '+str(error))
+
+def remote_tree(repo: str, ref: str) -> dict:
+    if not re.fullmatch(r'[a-f0-9]{40}', ref): raise ProbeError('A full pinned commit is required')
+    result = json.loads(request(f'https://api.github.com/repos/{repo}/git/trees/{ref}?recursive=1'))
+    if result.get('truncated'): raise ProbeError('GitHub returned truncated tree; refusing incomplete source set')
+    if not isinstance(result.get('tree'), list): raise ProbeError('Missing git tree')
+    return result
+
+def enumerate_blobs(tree: dict, repo: str, ref: str, prefix: str = '', main: bool = True) -> list[dict]:
+    result = []
+    for entry in tree['tree']:
+        name = safe_relative(entry['path'])
+        if entry['type'] != 'blob' or entry.get('mode') not in ('100644','100755'): continue
+        if not selected(name, main): continue
+        if not isinstance(entry.get('size'), int) or not 0 <= entry['size'] <= MAX_BLOB:
+            raise ProbeError('Unsupported source size: '+name)
+        result.append({'path':prefix+name, 'remote_path':name, 'repository':repo,
+                       'commit':ref, 'git_blob_sha1':entry['sha'], 'size':entry['size']})
+    return result
+
+def fetch(destination: Path) -> dict:
+    destination.mkdir(parents=True, exist_ok=True)
+    tree = remote_tree(REPOSITORY, COMMIT)
+    write_json(destination/'upstream-tree.json', tree)
+    entries = enumerate_blobs(tree, REPOSITORY, COMMIT)
+    modules=[]
+    for item in tree['tree']:
+        if item['type'] != 'commit' or not item['path'].startswith('src/'): continue
+        name=safe_relative(item['path']); repo=SUBMODULES.get(name)
+        if not repo: raise ProbeError('Unconfigured source submodule: '+name)
+        child=remote_tree(repo,item['sha'])
+        modules.append({'path':name,'repository':repo,'commit':item['sha']})
+        entries.extend(enumerate_blobs(child,repo,item['sha'],name+'/',False))
+    names=[e['path'] for e in entries]
+    if len(names)!=len(set(names)): raise ProbeError('Overlapping source paths')
+    if not entries: raise ProbeError('No source blobs selected')
+    def obtain(e: dict) -> dict:
+        target=destination/e['path'];target.parent.mkdir(parents=True,exist_ok=True)
+        if target.is_symlink(): raise ProbeError('Symlink in source tree: '+str(target))
+        if target.exists():
+            data=target.read_bytes()
+            if git_blob(data)!=e['git_blob_sha1']:
+                raise ProbeError('Modified or corrupt baseline: '+str(target)+'; move it aside before refetch')
+        else:
+            rawpath=urllib.parse.quote(e['remote_path'],safe='/')
+            data=request(f"https://raw.githubusercontent.com/{e['repository']}/{e['commit']}/{rawpath}",MAX_BLOB)
+            if b'\0' in data: raise ProbeError('Non-text source blob rejected: '+e['path'])
+            if len(data)!=e['size'] or git_blob(data)!=e['git_blob_sha1']:
+                raise ProbeError('Source identity mismatch: '+e['path'])
+            temp=target.with_name(target.name+'.download');temp.write_bytes(data);os.replace(temp,target)
+        return {**e,'sha256':hashlib.sha256(data).hexdigest()}
+    done=[]
+    with concurrent.futures.ThreadPoolExecutor(max_workers=6) as pool:
+        for e in pool.map(obtain,entries):
+            done.append(e)
+            if len(done)%100==0: print(f'Source only: {len(done)}/{len(entries)} files verified',flush=True)
+    receipt={'schema':'kam-source-receipt/1','commit':COMMIT,'repository':REPOSITORY,
+             'downloaded_at':stamp(),'source_only':True,'submodules':modules,
+             'files':sorted(done,key=lambda e:e['path'])}
+    write_json(destination/'SOURCE_RECEIPT.json',receipt)
+    verify(destination)
+    return receipt
+
+def verify(destination: Path) -> dict:
+    file=destination/'SOURCE_RECEIPT.json'
+    if not file.is_file(): raise ProbeError('Source receipt missing; run tools/source_fetch.py')
+    d=json.loads(file.read_text())
+    if d.get('commit')!=COMMIT or d.get('repository')!=REPOSITORY or not d.get('source_only'):
+        raise ProbeError('Wrong source baseline')
+    expected=set()
+    for item in d.get('files',[]):
+        name=safe_relative(item['path']);p=destination/name
+        if name in expected: raise ProbeError('Duplicate source receipt entry')
+        expected.add(name)
+        if p.is_symlink() or not p.is_file(): raise ProbeError('Source missing/linked: '+name)
+        if p.stat().st_size!=item['size'] or sha256(p)!=item['sha256'] or git_blob(p.read_bytes())!=item['git_blob_sha1']:
+            raise ProbeError('Source modified: '+name)
+    if not expected: raise ProbeError('Empty source set')
+    for p in destination.rglob('*'):
+        if p.is_symlink(): raise ProbeError('Symlink in source baseline: '+str(p))
+        if p.is_file() and p.suffix.lower() in (CODE|{'.ppu','.o','.a','.so','.dll','.res'}):
+            if p.relative_to(destination).as_posix() not in expected:
+                raise ProbeError('Unrecorded source/binary in baseline: '+str(p))
+    return d
+
+def import_local(source: Path, destination: Path) -> dict:
+    """Import an ALREADY pinned code-only receipt tree; no arbitrary git tree trusted."""
+    import shutil
+    d=verify(source)
+    if destination.exists() and any(destination.iterdir()): raise ProbeError('Destination must be empty')
+    destination.mkdir(parents=True,exist_ok=True)
+    for item in d['files']:
+        p=destination/item['path'];p.parent.mkdir(parents=True,exist_ok=True)
+        shutil.copy2(source/item['path'],p)
+    write_json(destination/'SOURCE_RECEIPT.json',d)
+    return verify(destination)
+
+if __name__=='__main__':
+    ap=argparse.ArgumentParser(description=__doc__)
+    ap.add_argument('--verify',action='store_true');ap.add_argument('--destination',type=Path,default=ROOT/'vendor/kam_remake')
+    a=ap.parse_args()
+    try:
+        d=verify(a.destination) if a.verify else fetch(a.destination)
+        print(f"Verified {len(d['files'])} source-text files, commit {COMMIT}")
+    except (ProbeError,OSError,ValueError,KeyError) as e:
+        print('SOURCE BLOCKED:',e);raise SystemExit(2)