148 lines
8.2 KiB
Python
Executable File
148 lines
8.2 KiB
Python
Executable File
"""Fetch only source-text blobs at r16020. Never clone or download release archives.
|
|
On the server, downloads Pascal/includes and dependency source, NOT data/maps/sprites,
|
|
DLLs, .so, .a, .o, compiled resources, executables, or history. Requires Python stdlib.
|
|
"""
|
|
from __future__ import annotations
|
|
import argparse, concurrent.futures, hashlib, json, os, re, time, urllib.error, urllib.parse, urllib.request
|
|
from pathlib import Path, PurePosixPath
|
|
from common import ROOT, COMMIT, REPOSITORY, ProbeError, git_blob, safe_relative, write_json, sha256, stamp
|
|
|
|
CODE = {'.pas', '.pp', '.inc', '.inl', '.h', '.hpp', '.c', '.cpp', '.s', '.asm'}
|
|
SKIP_PARTS = {'examples','example','demos','demo','tests','test','docs','doc','.git'}
|
|
MAX_BLOB = 16 * 1024 * 1024
|
|
SUBMODULES = {'src/ext/pascalscript_submodule': 'reyandme/pascalscript'}
|
|
|
|
def selected(path: str, main: bool = True) -> bool:
|
|
safe_relative(path)
|
|
p = PurePosixPath(path)
|
|
if any(x.casefold() in SKIP_PARTS for x in p.parts[:-1]): return False
|
|
license_name = p.name.casefold().startswith(('license','licence','copying','copyright'))
|
|
if license_name: return p.suffix.casefold() in ('', '.txt', '.md', '.rst')
|
|
if p.suffix.casefold() not in CODE: return False
|
|
return not main or path.startswith('src/') or '/' not in path
|
|
|
|
def request(url: str, max_bytes: int = 32*1024*1024) -> bytes:
|
|
headers = {'User-Agent': 'KaM-Android-Architecture-Probe/0.3', 'Accept':'application/vnd.github+json'}
|
|
# Never send a token to the raw-content host, never print tokens.
|
|
token = os.environ.get('GITHUB_TOKEN')
|
|
if token and urllib.parse.urlparse(url).hostname == 'api.github.com':
|
|
headers['Authorization'] = 'Bearer ' + token
|
|
error = None
|
|
for attempt in range(3):
|
|
try:
|
|
with urllib.request.urlopen(urllib.request.Request(url, headers=headers), timeout=60) as r:
|
|
result = r.read(max_bytes+1)
|
|
if len(result) > max_bytes: raise ProbeError('Oversized source response: '+url)
|
|
return result
|
|
except (urllib.error.URLError, TimeoutError) as e:
|
|
error = e
|
|
if isinstance(e, urllib.error.HTTPError) and e.code in (401,403,404): break
|
|
time.sleep(attempt+1)
|
|
raise ProbeError('Source request failed: '+url+'; '+str(error))
|
|
|
|
def remote_tree(repo: str, ref: str) -> dict:
|
|
if not re.fullmatch(r'[a-f0-9]{40}', ref): raise ProbeError('A full pinned commit is required')
|
|
result = json.loads(request(f'https://api.github.com/repos/{repo}/git/trees/{ref}?recursive=1'))
|
|
if result.get('truncated'): raise ProbeError('GitHub returned truncated tree; refusing incomplete source set')
|
|
if not isinstance(result.get('tree'), list): raise ProbeError('Missing git tree')
|
|
return result
|
|
|
|
def enumerate_blobs(tree: dict, repo: str, ref: str, prefix: str = '', main: bool = True) -> list[dict]:
|
|
result = []
|
|
for entry in tree['tree']:
|
|
name = safe_relative(entry['path'])
|
|
if entry['type'] != 'blob' or entry.get('mode') not in ('100644','100755'): continue
|
|
if not selected(name, main): continue
|
|
if not isinstance(entry.get('size'), int) or not 0 <= entry['size'] <= MAX_BLOB:
|
|
raise ProbeError('Unsupported source size: '+name)
|
|
result.append({'path':prefix+name, 'remote_path':name, 'repository':repo,
|
|
'commit':ref, 'git_blob_sha1':entry['sha'], 'size':entry['size']})
|
|
return result
|
|
|
|
def fetch(destination: Path) -> dict:
|
|
destination.mkdir(parents=True, exist_ok=True)
|
|
tree = remote_tree(REPOSITORY, COMMIT)
|
|
write_json(destination/'upstream-tree.json', tree)
|
|
entries = enumerate_blobs(tree, REPOSITORY, COMMIT)
|
|
modules=[]
|
|
for item in tree['tree']:
|
|
if item['type'] != 'commit' or not item['path'].startswith('src/'): continue
|
|
name=safe_relative(item['path']); repo=SUBMODULES.get(name)
|
|
if not repo: raise ProbeError('Unconfigured source submodule: '+name)
|
|
child=remote_tree(repo,item['sha'])
|
|
modules.append({'path':name,'repository':repo,'commit':item['sha']})
|
|
entries.extend(enumerate_blobs(child,repo,item['sha'],name+'/',False))
|
|
names=[e['path'] for e in entries]
|
|
if len(names)!=len(set(names)): raise ProbeError('Overlapping source paths')
|
|
if not entries: raise ProbeError('No source blobs selected')
|
|
def obtain(e: dict) -> dict:
|
|
target=destination/e['path'];target.parent.mkdir(parents=True,exist_ok=True)
|
|
if target.is_symlink(): raise ProbeError('Symlink in source tree: '+str(target))
|
|
if target.exists():
|
|
data=target.read_bytes()
|
|
if git_blob(data)!=e['git_blob_sha1']:
|
|
raise ProbeError('Modified or corrupt baseline: '+str(target)+'; move it aside before refetch')
|
|
else:
|
|
rawpath=urllib.parse.quote(e['remote_path'],safe='/')
|
|
data=request(f"https://raw.githubusercontent.com/{e['repository']}/{e['commit']}/{rawpath}",MAX_BLOB)
|
|
if b'\0' in data: raise ProbeError('Non-text source blob rejected: '+e['path'])
|
|
if len(data)!=e['size'] or git_blob(data)!=e['git_blob_sha1']:
|
|
raise ProbeError('Source identity mismatch: '+e['path'])
|
|
temp=target.with_name(target.name+'.download');temp.write_bytes(data);os.replace(temp,target)
|
|
return {**e,'sha256':hashlib.sha256(data).hexdigest()}
|
|
done=[]
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=6) as pool:
|
|
for e in pool.map(obtain,entries):
|
|
done.append(e)
|
|
if len(done)%100==0: print(f'Source only: {len(done)}/{len(entries)} files verified',flush=True)
|
|
receipt={'schema':'kam-source-receipt/1','commit':COMMIT,'repository':REPOSITORY,
|
|
'downloaded_at':stamp(),'source_only':True,'submodules':modules,
|
|
'files':sorted(done,key=lambda e:e['path'])}
|
|
write_json(destination/'SOURCE_RECEIPT.json',receipt)
|
|
verify(destination)
|
|
return receipt
|
|
|
|
def verify(destination: Path) -> dict:
|
|
file=destination/'SOURCE_RECEIPT.json'
|
|
if not file.is_file(): raise ProbeError('Source receipt missing; run tools/source_fetch.py')
|
|
d=json.loads(file.read_text())
|
|
if d.get('commit')!=COMMIT or d.get('repository')!=REPOSITORY or not d.get('source_only'):
|
|
raise ProbeError('Wrong source baseline')
|
|
expected=set()
|
|
for item in d.get('files',[]):
|
|
name=safe_relative(item['path']);p=destination/name
|
|
if name in expected: raise ProbeError('Duplicate source receipt entry')
|
|
expected.add(name)
|
|
if p.is_symlink() or not p.is_file(): raise ProbeError('Source missing/linked: '+name)
|
|
if p.stat().st_size!=item['size'] or sha256(p)!=item['sha256'] or git_blob(p.read_bytes())!=item['git_blob_sha1']:
|
|
raise ProbeError('Source modified: '+name)
|
|
if not expected: raise ProbeError('Empty source set')
|
|
for p in destination.rglob('*'):
|
|
if p.is_symlink(): raise ProbeError('Symlink in source baseline: '+str(p))
|
|
if p.is_file() and p.suffix.lower() in (CODE|{'.ppu','.o','.a','.so','.dll','.res'}):
|
|
if p.relative_to(destination).as_posix() not in expected:
|
|
raise ProbeError('Unrecorded source/binary in baseline: '+str(p))
|
|
return d
|
|
|
|
def import_local(source: Path, destination: Path) -> dict:
|
|
"""Import an ALREADY pinned code-only receipt tree; no arbitrary git tree trusted."""
|
|
import shutil
|
|
d=verify(source)
|
|
if destination.exists() and any(destination.iterdir()): raise ProbeError('Destination must be empty')
|
|
destination.mkdir(parents=True,exist_ok=True)
|
|
for item in d['files']:
|
|
p=destination/item['path'];p.parent.mkdir(parents=True,exist_ok=True)
|
|
shutil.copy2(source/item['path'],p)
|
|
write_json(destination/'SOURCE_RECEIPT.json',d)
|
|
return verify(destination)
|
|
|
|
if __name__=='__main__':
|
|
ap=argparse.ArgumentParser(description=__doc__)
|
|
ap.add_argument('--verify',action='store_true');ap.add_argument('--destination',type=Path,default=ROOT/'vendor/kam_remake')
|
|
a=ap.parse_args()
|
|
try:
|
|
d=verify(a.destination) if a.verify else fetch(a.destination)
|
|
print(f"Verified {len(d['files'])} source-text files, commit {COMMIT}")
|
|
except (ProbeError,OSError,ValueError,KeyError) as e:
|
|
print('SOURCE BLOCKED:',e);raise SystemExit(2)
|