commit dfcc43c72d3fd62cbbc060fd0dcf10bf23d3fabc Author: admin_d Date: Fri Sep 11 18:18:40 2026 +0500 Initial import diff --git a/.gitignore b/.gitignore new file mode 100755 index 0000000..8ebf5e8 --- /dev/null +++ b/.gitignore @@ -0,0 +1,20 @@ +# Source-only repository: never commit generated outputs or external assets. +/vendor/ +/private-assets/ +/build/ +/dist/ +/.cache/ +/reports/ +/docs/ +/assets/ +/data/ +/config/toolchain.lock.json +**/__pycache__/ +*.md +*.txt +*.sha256 +*.apk +*.so +*.a +*.ppu +*.o diff --git a/build-android.sh b/build-android.sh new file mode 100755 index 0000000..a0bf937 --- /dev/null +++ b/build-android.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +cd "$ROOT" +exec python3 tools/runner.py "$@" diff --git a/code/DecisionHostCallback.pas b/code/DecisionHostCallback.pas new file mode 100755 index 0000000..f8183bc --- /dev/null +++ b/code/DecisionHostCallback.pas @@ -0,0 +1,22 @@ +unit DecisionHostCallback; +{$mode objfpc}{$H+} +{$ifndef ANDROID}{$fatal Android-only decision host probe}{$endif} +interface +implementation +uses SysUtils,fpjson,CastleWindow,ProbeReport,ProbePlatform,ProbeDecision; +var Window:TCastleWindow; +procedure Initialize; +var R:TProbeReport;Dir:string; +begin + R:=TProbeReport.Create; + try + Dir:=ProbeDirectory;R.Root.Add('report_directory',Dir); + R.Add('android.startup','PASS','Application.OnInitialize reached without a custom TCastleView'); + R.Add('decision.host.callback','PASS','Lifecycle application callback host variant executed'); + RunPlatformChecks(R,Dir);RunDecisionChecks(R,Dir);R.Save(Dir+'runtime.json'); + finally R.Free;end; +end; +initialization + Application.OnInitialize:=@Initialize; + Window:=TCastleWindow.Create(Application);Application.MainWindow:=Window; +end. diff --git a/code/DecisionHostText.pas b/code/DecisionHostText.pas new file mode 100755 index 0000000..7460fb1 --- /dev/null +++ b/code/DecisionHostText.pas @@ -0,0 +1,22 @@ +unit DecisionHostText; +{$mode objfpc}{$H+} +{$ifndef ANDROID}{$fatal Android-only text provider probe}{$endif} +interface +implementation +uses cwstring,fpjson,CastleWindow,ProbeReport,ProbePlatform,ProbeTextProvider; +var Window:TCastleWindow; +procedure Initialize; +var R:TProbeReport;Dir:string; +begin + R:=TProbeReport.Create; + try + Dir:=ProbeDirectory;R.Root.Add('report_directory',Dir); + R.Add('android.startup','PASS','Pinned FPC text-provider host initialized'); + R.Add('decision.text.cwstring_unit','PASS','cwstring unit linked and initialized for Android target'); + RunTextProviderChecks(R);R.Save(Dir+'runtime.json'); + finally R.Free;end; +end; +initialization + Application.OnInitialize:=@Initialize; + Window:=TCastleWindow.Create(Application);Application.MainWindow:=Window; +end. diff --git a/code/DecisionHostView.pas b/code/DecisionHostView.pas new file mode 100755 index 0000000..1e2dab9 --- /dev/null +++ b/code/DecisionHostView.pas @@ -0,0 +1,27 @@ +unit DecisionHostView; +{$mode objfpc}{$H+} +{$ifndef ANDROID}{$fatal Android-only decision host probe}{$endif} +interface +implementation +uses SysUtils,fpjson,CastleWindow,CastleUIControls,ProbeReport,ProbePlatform,ProbeDecision; +type + TDecisionView=class(TCastleView) + public procedure Start;override;end; +var Window:TCastleWindow; +procedure TDecisionView.Start; +var R:TProbeReport;Dir:string; +begin + inherited;R:=TProbeReport.Create; + try + Dir:=ProbeDirectory;R.Root.Add('report_directory',Dir); + R.Add('android.startup','PASS','Minimal TCastleView Start reached without child controls'); + R.Add('decision.host.view','PASS','TCastleView host variant reached Start'); + RunPlatformChecks(R,Dir);RunDecisionChecks(R,Dir);R.Save(Dir+'runtime.json'); + finally R.Free;end; +end; +procedure Initialize; +begin Window.Container.View:=TDecisionView.Create(Application);end; +initialization + Application.OnInitialize:=@Initialize; + Window:=TCastleWindow.Create(Application);Application.MainWindow:=Window; +end. diff --git a/code/GameInitialize.pas b/code/GameInitialize.pas new file mode 100755 index 0000000..660c43d --- /dev/null +++ b/code/GameInitialize.pas @@ -0,0 +1,16 @@ +unit GameInitialize; +{$mode objfpc}{$H+} +{$ifndef ANDROID}{$fatal Android-only architecture probe; there is no desktop target}{$endif} +interface +implementation +uses CastleWindow, GameViewMain; +var Window: TCastleWindow; +procedure Initialize; +begin + Window.Container.View := TProbeView.Create(Application); +end; +initialization + Application.OnInitialize := @Initialize; + Window := TCastleWindow.Create(Application); + Application.MainWindow := Window; +end. diff --git a/code/GameViewMain.pas b/code/GameViewMain.pas new file mode 100755 index 0000000..58ee6f1 --- /dev/null +++ b/code/GameViewMain.pas @@ -0,0 +1,241 @@ +unit GameViewMain; +{$mode objfpc}{$H+}{$codepage utf8}{$pointermath on} +interface +uses Classes, SysUtils, CastleUIControls, CastleControls, CastleKeysMouse, + CastleVectors, CastleSoundEngine, ProbeReport, ProbeCamera, ProbeScheduling; +type + TProbeView = class(TCastleView) + private + FStatus,FHelp:TCastleLabel; + FImage:TCastleImageControl; + FSound:TCastleSound; + FReport:TProbeReport; + FDir,FCell:string; + FCamera:TProbeCamera; + FDown:array[0..9] of Boolean; + FPos,FStart:array[0..9] of TVector2; + FMoved,FMulti,FStarted,FListeners:Boolean; + FTaps,FPans,FPinches,FFrames,FUpdates,FPauses,FResumes,FGLCloses,FGLOpens:Integer; + FElapsed:Single; + FSchedule:TDecisionSchedule; + procedure MakeImage; + procedure PlaceImage; + procedure SaveReport; + procedure SaveClicked(Sender:TObject); + procedure SoundClicked(Sender:TObject); + procedure Paused(Sender:TObject); + procedure Resumed(Sender:TObject); + procedure ContextOpened(Sender:TObject); + procedure ContextClosed(Sender:TObject); + function Fingers(out A,B:Integer):Integer; + public + procedure Start; override; + procedure Stop; override; + procedure Render; override; + procedure Update(const SecondsPassed:Single;var HandleInput:Boolean); override; + function Press(const Event:TInputPressRelease):Boolean; override; + function Release(const Event:TInputPressRelease):Boolean; override; + function Motion(const Event:TInputMotion):Boolean; override; + end; +implementation +uses Math, cwstring, fpjson, CastleImages, CastleUriUtils, CastleApplicationProperties, + CastleLog, ProbePlatform, ProbeAssets, ProbeDecision, ProbeTextProvider, ProbeImports, ProbeBuild; +procedure TProbeView.Start; +var B:TCastleButton; DataDir:string; +begin + inherited; + FImage:=TCastleImageControl.Create(FreeAtStop);FImage.Stretch:=True;InsertFront(FImage); + FStatus:=TCastleLabel.Create(FreeAtStop);FStatus.FontSize:=16; + FStatus.Anchor(hpLeft,12);FStatus.Anchor(vpTop,-10);InsertFront(FStatus); + FHelp:=TCastleLabel.Create(FreeAtStop);FHelp.FontSize:=14; + FHelp.Anchor(hpLeft,12);FHelp.Anchor(vpBottom,10);InsertFront(FHelp); + B:=TCastleButton.Create(FreeAtStop);B.Caption:='Save report'; + B.Anchor(hpLeft,12);B.Anchor(vpTop,-90);B.OnClick:=@SaveClicked;InsertFront(B); + B:=TCastleButton.Create(FreeAtStop);B.Caption:='Test sound'; + B.Anchor(hpLeft,175);B.Anchor(vpTop,-90);B.OnClick:=@SoundClicked;InsertFront(B); + FSound:=TCastleSound.Create(FreeAtStop); + FReport:=TProbeReport.Create; + try + FDir:=ProbeDirectory; + FReport.Root.Add('report_directory',FDir); + FReport.Add('android.startup','PASS','CGE application initialized; library and unit initializers reached UI'); + RunPlatformChecks(FReport,FDir); + RunDecisionChecks(FReport,FDir); + FReport.Add('decision.text.cwstring_unit','PASS','cwstring unit linked and initialized in the common Android runtime APK'); + RunTextProviderChecks(FReport); + RunUpstreamChecks(FReport); + FSchedule:=TDecisionSchedule.Create(FReport); + try DataDir:=PrepareOptionalAssets(FReport,FDir); + FReport.Root.Add('native_asset_directory',DataDir); + except on E:Exception do FReport.Add('assets.native_adapter','FAIL',E.ClassName+': '+E.Message);end; + MakeImage; + FCamera.Reset; + FCamera.PixelsPerCell:=Max(4.0,Min((Container.UnscaledWidth-40)/32, + (Container.UnscaledHeight-180)/32)); + FCamera.OffsetY:=40;PlaceImage; + FReport.Add('render.visual_correctness','NOT_RUN','Procedural checker only; inspect on device. Not KaM terrain.'); + FReport.Add('audio.audible','NOT_RUN','Press Test sound and listen; API success does not prove audible output'); + ApplicationProperties.OnPause.Add(@Paused); + ApplicationProperties.OnResume.Add(@Resumed); + ApplicationProperties.OnGLContextOpenObject.Add(@ContextOpened); + ApplicationProperties.OnGLContextCloseObject.Add(@ContextClosed); + FListeners:=True; + FReport.Add('render.context','NOT_RUN','Awaiting the first render callback'); + FStarted:=True; + FStatus.Caption:='Android / '+ProbeProfile+' / '+ProbeCpu+LineEnding+ + 'Automatic smoke: '+FReport.Status+'; device observations separate'+LineEnding+ + 'Architecture probe only. No KaM world or simulation.'; + SaveReport; + except on E:Exception do begin + FReport.Add('android.startup','FAIL',E.ClassName+': '+E.Message); + FStatus.Caption:='STARTUP FAILED: '+E.Message; + if FDir<>'' then SaveReport; + end;end; +end; +procedure TProbeView.MakeImage; +var Image:TRGBAlphaImage;Row:PByte;X,Y:Integer;V:Byte; +begin + Image:=TRGBAlphaImage.Create(256,256); + try + for Y:=0 to 255 do begin + Row:=PByte(Image.RowPtr(Y)); + for X:=0 to 255 do begin + if ((X div 8)+(Y div 8)) mod 2=0 then V:=90 else V:=150; + Row[X*4]:=V;Row[X*4+1]:=V;Row[X*4+2]:=V;Row[X*4+3]:=255; + end; + end; + FImage.Image:=Image;Image:=nil; + finally Image.Free;end; +end; +procedure TProbeView.PlaceImage; +begin + FImage.Left:=FCamera.OffsetX;FImage.Bottom:=FCamera.OffsetY; + FImage.Width:=32*FCamera.PixelsPerCell;FImage.Height:=32*FCamera.PixelsPerCell; + FHelp.Caption:=Format('Synthetic grid | tap %d | pan %d | pinch %d | cell %s', + [FTaps,FPans,FPinches,FCell]); +end; +procedure TProbeView.SaveReport; +var J:TJSONObject; +begin + if (FReport=nil) or (FDir='') then Exit; + try + FReport.Root.Delete('session');J:=TJSONObject.Create; + J.Add('frames',FFrames);J.Add('updates',FUpdates); + J.Add('taps',FTaps);J.Add('pan_events',FPans);J.Add('pinch_events',FPinches); + J.Add('pauses',FPauses);J.Add('resumes',FResumes); + J.Add('gl_open_events',FGLOpens);J.Add('gl_close_events',FGLCloses); + J.Add('last_cell',FCell);J.Add('width',Container.UnscaledWidth); + J.Add('height',Container.UnscaledHeight); + FReport.Root.Add('session',J); + if FFrames>0 then FReport.Check('render.context',ApplicationProperties.IsGLContextOpen,'CGE GL context during render/update'); + if FFrames>=30 then FReport.Add('render.frame_callbacks','PASS','At least 30 render callbacks; visual output still requires observation'); + FReport.Save(FDir+'runtime.json'); + WritelnLog('KaMProbe','KAM_PROBE_REPORT '+FDir+'runtime.json'); + except on E:Exception do begin + WritelnLog('KaMProbe','REPORT_SAVE_FAILED '+E.Message); + FStatus.Caption:='REPORT SAVE FAILED: '+E.Message; + end;end; +end; +procedure TProbeView.SaveClicked(Sender:TObject); +begin SaveReport;FHelp.Caption:='Report: '+FDir+'runtime.json';end; +procedure TProbeView.SoundClicked(Sender:TObject); +begin + try + WriteTestTone(FDir+'generated-test-tone.wav'); + FSound.Url:=FilenameToUriSafe(FDir+'generated-test-tone.wav'); + SoundEngine.Play(FSound); + FReport.Add('audio.api','OBSERVED','CGE Play returned without exception; confirm audibility manually'); + except on E:Exception do FReport.Add('audio.api','FAIL',E.ClassName+': '+E.Message);end; + SaveReport; +end; +procedure TProbeView.Paused(Sender:TObject); +begin Inc(FPauses);if FSchedule<>nil then FSchedule.Paused;FillChar(FDown,SizeOf(FDown),0);FMoved:=True;SaveReport;end; +procedure TProbeView.Resumed(Sender:TObject); +begin Inc(FResumes);if FSchedule<>nil then FSchedule.Resumed;SaveReport;end; +procedure TProbeView.ContextOpened(Sender:TObject); +begin Inc(FGLOpens);end; +procedure TProbeView.ContextClosed(Sender:TObject); +begin Inc(FGLCloses);SaveReport;end; +procedure TProbeView.Stop; +begin + if FListeners then begin + ApplicationProperties.OnPause.Remove(@Paused); + ApplicationProperties.OnResume.Remove(@Resumed); + ApplicationProperties.OnGLContextOpenObject.Remove(@ContextOpened); + ApplicationProperties.OnGLContextCloseObject.Remove(@ContextClosed); + FListeners:=False; + end; + SaveReport;FStarted:=False; + FreeAndNil(FSchedule); + FillChar(FDown,SizeOf(FDown),0);FreeAndNil(FReport); + inherited; +end; +procedure TProbeView.Render; +begin inherited;Inc(FFrames);end; +procedure TProbeView.Update(const SecondsPassed:Single;var HandleInput:Boolean); +begin + inherited;Inc(FUpdates); + if FSchedule<>nil then FSchedule.Tick; + FElapsed:=FElapsed+SecondsPassed; + if FStarted and (FElapsed>=2) then begin SaveReport;FElapsed:=0;end; +end; +function TProbeView.Fingers(out A,B:Integer):Integer; +var I:Integer; +begin + Result:=0;A:=-1;B:=-1; + for I:=0 to High(FDown) do if FDown[I] then begin + Inc(Result);if A<0 then A:=I else if B<0 then B:=I; + end; +end; +function TProbeView.Press(const Event:TInputPressRelease):Boolean; +var A,B,I:Integer; +begin + Result:=inherited;if Result or not FStarted then Exit; + if not Event.IsMouseButton(buttonLeft) then Exit; + I:=Event.FingerIndex;if (I<0) or (I>High(FDown)) then Exit; + if Fingers(A,B)=0 then begin FMoved:=False;FMulti:=False;end; + FDown[I]:=True;FPos[I]:=Event.Position;FStart[I]:=Event.Position; + if Fingers(A,B)>1 then FMulti:=True; + Result:=True; +end; +function TProbeView.Motion(const Event:TInputMotion):Boolean; +var I,N,A,B:Integer;OldDistance,NewDistance:Single;OldMid,NewMid:TVector2; +begin + Result:=inherited; + I:=Event.FingerIndex;if (I<0) or (I>High(FDown)) or not FDown[I] then Exit; + N:=Fingers(A,B); + if N>=2 then begin + OldDistance:=Sqrt(Sqr(FPos[A].X-FPos[B].X)+Sqr(FPos[A].Y-FPos[B].Y)); + OldMid:=(FPos[A]+FPos[B])*0.5; + end; + if Sqr(Event.Position.X-FStart[I].X)+Sqr(Event.Position.Y-FStart[I].Y)>64 then FMoved:=True; + if N=1 then begin + { Do not pan before the movement threshold: a tap must not shift the grid. } + if FMoved then begin + FCamera.Pan(Event.Position.X-FPos[I].X,Event.Position.Y-FPos[I].Y);Inc(FPans); + end; + end; + FPos[I]:=Event.Position; + if N>=2 then begin + NewDistance:=Sqrt(Sqr(FPos[A].X-FPos[B].X)+Sqr(FPos[A].Y-FPos[B].Y)); + NewMid:=(FPos[A]+FPos[B])*0.5; + if (OldDistance>1) and (NewDistance>1) then FCamera.ZoomAt(NewDistance/OldDistance,OldMid.X,OldMid.Y); + FCamera.Pan(NewMid.X-OldMid.X,NewMid.Y-OldMid.Y);Inc(FPinches); + end; + PlaceImage;Result:=True; +end; +function TProbeView.Release(const Event:TInputPressRelease):Boolean; +var I,N,A,B,X,Y:Integer; +begin + Result:=inherited; + if not Event.IsMouseButton(buttonLeft) then Exit; + I:=Event.FingerIndex;if (I<0) or (I>High(FDown)) or not FDown[I] then Exit; + N:=Fingers(A,B);FDown[I]:=False; + if (N=1) and not FMoved and not FMulti then begin + Inc(FTaps); + if FCamera.CellAt(Event.Position.X,Event.Position.Y,32,32,X,Y) then + FCell:=IntToStr(X)+','+IntToStr(Y) else FCell:='outside'; + end; + PlaceImage;SaveReport;Result:=True; +end; +end. diff --git a/code/ProbeAssets.pas b/code/ProbeAssets.pas new file mode 100755 index 0000000..e2e341e --- /dev/null +++ b/code/ProbeAssets.pas @@ -0,0 +1,55 @@ +unit ProbeAssets; +{$mode objfpc}{$H+} +interface +uses ProbeReport; +function PrepareOptionalAssets(Report:TProbeReport;const WorkDir:string):string; +implementation +uses Classes,SysUtils,fpjson,jsonparser,CastleDownload,CastleFilesUtils; +function SafeRelative(const S:string):Boolean; +var Parts:TStringList; I:Integer; +begin + Result:=False; + if (S='') or (S[1]='/') or (Pos('\',S)>0) or (Pos(':',S)>0) + or (Pos(#0,S)>0) or (Pos(#10,S)>0) or (Pos(#13,S)>0) then Exit; + Parts:=TStringList.Create; + try + Parts.StrictDelimiter:=True;Parts.Delimiter:='/';Parts.DelimitedText:=S; + for I:=0 to Parts.Count-1 do + if (Parts[I]='') or (Parts[I]='.') or (Parts[I]='..') then Exit; + Result:=True; + finally Parts.Free;end; +end; +function PrepareOptionalAssets(Report:TProbeReport;const WorkDir:string):string; +var Manifest:TJSONData; Files:TJSONArray; I:Integer; Rel,Name,Url:string; + Src:TStream;Dst:TFileStream;Count,Expected:Int64; +begin + Result:=IncludeTrailingPathDelimiter(WorkDir)+'game/'; + Manifest:=GetJSON(FileToString('castle-data:/assets-manifest.json')); + try + Files:=TJSONObject(Manifest).Arrays['files']; + if Files.Count=0 then begin + Report.Add('assets.native_adapter','NOT_RUN','No game resources supplied; code-only build is expected'); + Exit; + end; + Count:=0; + for I:=0 to Files.Count-1 do begin + Rel:=Files.Objects[I].Strings['path']; + if not SafeRelative(Rel) then raise Exception.Create('Unsafe asset name: '+Rel); + Name:=Result+Rel; Url:=Files.Objects[I].Strings['url']; + if Pos('castle-data:/game/',Url)<>1 then raise Exception.Create('Unsupported asset URL'); + Expected:=Files.Objects[I].Int64s['size']; + if not ForceDirectories(ExtractFileDir(Name)) then raise Exception.Create('Cannot create asset directory'); + Src:=Download(Url); + try + Dst:=TFileStream.Create(Name,fmCreate); + try + Dst.CopyFrom(Src,0); + if Dst.Size<>Expected then raise Exception.Create('Size mismatch: '+Rel); + Count:=Count+Dst.Size; + finally Dst.Free;end; + finally Src.Free;end; + end; + Report.Add('assets.native_adapter','PASS',Format('%d files, %d bytes copied to native paths; formats unchanged; NOT full loader proof',[Files.Count,Count])); + finally Manifest.Free;end; +end; +end. diff --git a/code/ProbeCamera.pas b/code/ProbeCamera.pas new file mode 100755 index 0000000..08dc98b --- /dev/null +++ b/code/ProbeCamera.pas @@ -0,0 +1,33 @@ +unit ProbeCamera; +{$mode delphi}{$H+} +{$ASSERTIONS ON}{$R+}{$Q+}{$B-} +interface +uses Math; +type + TProbeCamera = record + OffsetX, OffsetY, PixelsPerCell: Double; + procedure Reset; + procedure Pan(DX,DY: Double); + procedure ZoomAt(Factor,SX,SY: Double); + function CellAt(SX,SY: Double;Cols,Rows:Integer;out X,Y:Integer):Boolean; + end; +implementation +procedure TProbeCamera.Reset; +begin OffsetX:=20;OffsetY:=20;PixelsPerCell:=10;end; +procedure TProbeCamera.Pan(DX,DY:Double); +begin OffsetX:=OffsetX+DX;OffsetY:=OffsetY+DY;end; +procedure TProbeCamera.ZoomAt(Factor,SX,SY:Double); +var WX,WY,NewScale:Double; +begin + if (Factor<=0) or IsNan(Factor) or IsInfinite(Factor) then Exit; + WX:=(SX-OffsetX)/PixelsPerCell;WY:=(SY-OffsetY)/PixelsPerCell; + NewScale:=EnsureRange(PixelsPerCell*Factor,2.0,128.0); + PixelsPerCell:=NewScale;OffsetX:=SX-WX*NewScale;OffsetY:=SY-WY*NewScale; +end; +function TProbeCamera.CellAt(SX,SY:Double;Cols,Rows:Integer;out X,Y:Integer):Boolean; +begin + X:=Floor((SX-OffsetX)/PixelsPerCell)+1; + Y:=Rows-Floor((SY-OffsetY)/PixelsPerCell); + Result:=(X>=1) and (X<=Cols) and (Y>=1) and (Y<=Rows); +end; +end. diff --git a/code/ProbeDecision.pas b/code/ProbeDecision.pas new file mode 100755 index 0000000..df420a4 --- /dev/null +++ b/code/ProbeDecision.pas @@ -0,0 +1,120 @@ +unit ProbeDecision; +{$mode objfpc}{$H+}{$codepage utf8} +{$ifndef ANDROID}{$fatal Decision probes are Android-only}{$endif} +interface +uses ProbeReport; +procedure RunDecisionChecks(Report:TProbeReport;const WorkDir:string); +implementation +uses Classes,SysUtils,fpjson,jsonparser,CastleDownload,CastleFilesUtils; +procedure WriteAscii(const Name,Value:string); +var F:TFileStream; A:AnsiString; +begin + A:=AnsiString(Value);F:=TFileStream.Create(Name,fmCreate); + try if Length(A)>0 then F.WriteBuffer(A[1],Length(A));finally F.Free;end; +end; +function ReadAscii(const Name:string):string; +var F:TFileStream; A:AnsiString; +begin + Result:='';F:=TFileStream.Create(Name,fmOpenRead or fmShareDenyWrite); + try SetLength(A,F.Size);if Length(A)>0 then F.ReadBuffer(A[1],Length(A));Result:=string(A);finally F.Free;end; +end; +procedure ProbeRenameReplace(Report:TProbeReport;const WorkDir:string); +var Target,Temp,Detail:string; Ok:Boolean; +begin + Target:=WorkDir+'decision-rename-target.txt';Temp:=Target+'.tmp'; + try + WriteAscii(Target,'old');WriteAscii(Temp,'new'); + Ok:=RenameFile(Temp,Target); + if Ok then Ok:=(ReadAscii(Target)='new') and not FileExists(Temp); + Detail:='replace_existing=' + LowerCase(BoolToStr(Ok,True)); + if (not Ok) and FileExists(Temp) then DeleteFile(Temp); + Report.Add('decision.persistence.rename_replace','OBSERVED',Detail); + except on E:Exception do + Report.Add('decision.persistence.rename_replace','OBSERVED','replace_existing=false; error='+E.ClassName+': '+E.Message); + end; +end; +procedure ProbeAppend(Report:TProbeReport;const WorkDir:string); +var Name:string;F:TFileStream;A:AnsiString;Ok:Boolean; + procedure Append(const S:AnsiString); + begin + F:=TFileStream.Create(Name,fmOpenReadWrite or fmShareDenyNone); + try F.Position:=F.Size;if Length(S)>0 then F.WriteBuffer(S[1],Length(S));finally F.Free;end; + end; +begin + Name:=WorkDir+'decision-append.log'; + try + WriteAscii(Name,'A'+LineEnding);Append('B'+LineEnding);Append('C'+LineEnding); + A:=AnsiString(ReadAscii(Name)); + Ok:=A=AnsiString('A'+LineEnding+'B'+LineEnding+'C'+LineEnding); + Report.Add('decision.persistence.append_reopen','OBSERVED', + 'append_reopen_ordered='+LowerCase(BoolToStr(Ok,True))+'; bytes='+IntToStr(Length(A))); + except on E:Exception do + Report.Add('decision.persistence.append_reopen','OBSERVED','append_reopen_ordered=false; error='+E.ClassName+': '+E.Message); + end; +end; +function FirstAsset(out Rel,Url:string;out Expected:Int64):Boolean; +var Manifest:TJSONData;Files:TJSONArray; +begin + Result:=False;Rel:='';Url:='';Expected:=-1; + Manifest:=GetJSON(FileToString('castle-data:/assets-manifest.json')); + try + Files:=TJSONObject(Manifest).Arrays['files']; + if Files.Count=0 then Exit; + Rel:=Files.Objects[0].Strings['path']; + Url:=Files.Objects[0].Strings['url']; + Expected:=Files.Objects[0].Int64s['size']; + Result:=True; + finally Manifest.Free;end; +end; +procedure ProbeCastleStream(Report:TProbeReport); +var S:TStream;Rel,Url:string;Expected,SizeValue:Int64;N:Integer;B:array[0..31] of Byte;SizeOk,SeekOk:Boolean; +begin + if not FirstAsset(Rel,Url,Expected) then begin + Report.Add('decision.fixture.castle_stream','NOT_RUN','No server fixture selected; pass --assets and --asset-path');Exit; + end; + SizeOk:=False;SeekOk:=False;SizeValue:=-1; + try + S:=Download(Url); + try + try SizeValue:=S.Size;SizeOk:=SizeValue=Expected;except SizeOk:=False;end; + try + if Expected>SizeOf(B) then S.Position:=Expected div 2 else S.Position:=0; + N:=S.Read(B,SizeOf(B));SeekOk:=N>0; + except SeekOk:=False;end; + finally S.Free;end; + Report.Add('decision.fixture.castle_stream','OBSERVED', + Format('path=%s; expected_size=%d; size=%d; size_matches=%s; seek_read=%s', + [Rel,Expected,SizeValue,LowerCase(BoolToStr(SizeOk,True)),LowerCase(BoolToStr(SeekOk,True))])); + except on E:Exception do + Report.Add('decision.fixture.castle_stream','OBSERVED','path='+Rel+'; open=false; error='+E.ClassName+': '+E.Message); + end; +end; +procedure ProbeMaterialize(Report:TProbeReport;const WorkDir:string); +var S:TStream;F:TFileStream;Rel,Url,Name:string;Expected:Int64;Ok:Boolean; +begin + if not FirstAsset(Rel,Url,Expected) then begin + Report.Add('decision.fixture.materialized_native_path','NOT_RUN','No server fixture selected; pass --assets and --asset-path');Exit; + end; + Name:=WorkDir+'decision-materialized.dat';Ok:=False; + try + S:=Download(Url); + try + F:=TFileStream.Create(Name,fmCreate); + try F.CopyFrom(S,0);finally F.Free;end; + finally S.Free;end; + F:=TFileStream.Create(Name,fmOpenRead or fmShareDenyWrite); + try Ok:=F.Size=Expected;finally F.Free;end; + Report.Add('decision.fixture.materialized_native_path','OBSERVED', + 'source='+Rel+'; supported='+LowerCase(BoolToStr(Ok,True))+'; native_path='+Name+'; bytes='+IntToStr(Expected)); + except on E:Exception do + Report.Add('decision.fixture.materialized_native_path','OBSERVED','source='+Rel+'; supported=false; error='+E.ClassName+': '+E.Message); + end; +end; +procedure RunDecisionChecks(Report:TProbeReport;const WorkDir:string); +begin + ProbeRenameReplace(Report,WorkDir); + ProbeAppend(Report,WorkDir); + ProbeCastleStream(Report); + ProbeMaterialize(Report,WorkDir); +end; +end. diff --git a/code/ProbePlatform.pas b/code/ProbePlatform.pas new file mode 100755 index 0000000..12297f9 --- /dev/null +++ b/code/ProbePlatform.pas @@ -0,0 +1,131 @@ +unit ProbePlatform; +{$mode objfpc}{$H+}{$codepage utf8} +{$ifndef ANDROID}{$fatal This probe may only be compiled for Android}{$endif} +interface +uses Classes, SysUtils, ProbeReport; +function ProbeDirectory: string; +procedure RunPlatformChecks(Report: TProbeReport; const WorkDir: string); +procedure WriteTestTone(const FileName: string); +implementation +uses Math, ZStream, CastleUriUtils, ProbeBuild; +type + TSumThread = class(TThread) + public + Sum: Int64; + procedure Execute; override; + end; +procedure TSumThread.Execute; +var I: Integer; +begin + Sum := 0; + for I := 1 to 10000 do Sum := Sum + I; +end; +function ProbeDirectory: string; +begin + Result := UriToFilenameSafe(ResolveCastleConfigUrl('castle-config:/kam-architecture-probe/')); + if Result = '' then raise Exception.Create('No native path for castle-config'); + Result := IncludeTrailingPathDelimiter(Result); + if not ForceDirectories(Result) then raise Exception.Create('Cannot create private probe directory'); +end; +procedure RunPlatformChecks(Report: TProbeReport; const WorkDir: string); +var + B: array[0..4095] of Byte; + ReadBack: array[0..4095] of Byte; + I: Integer; N: LongInt; LE: Cardinal; + Name, S: string; + FileStream: TFileStream; + Raw, Compressed: TMemoryStream; + C: TCompressionStream; D: TDecompressionStream; + W: TSumThread; + Counter: TStringList; + Launch: Integer; +begin + LE := $04030201; + Report.Check('abi.pointer', SizeOf(Pointer) = ProbePointerBytes, + Format('pointer=%d integer=%d nativeint=%d extended=%d', + [SizeOf(Pointer),SizeOf(Integer),SizeOf(NativeInt),SizeOf(Extended)])); + Report.Check('abi.scalar_layout', (SizeOf(Integer)=4) and (SizeOf(Single)=4) + and (SizeOf(Double)=8) and (SizeOf(Int64)=8), 'Fixed-width scalar assumptions'); + Report.Check('abi.endianness', PByte(@LE)^=1, 'Little-endian native byte order'); + for I := 0 to High(B) do B[I] := Byte(I mod 251); + try + Name := WorkDir + 'native-I-O-тест.bin'; + FileStream := TFileStream.Create(Name, fmCreate); + try FileStream.WriteBuffer(B, SizeOf(B)); finally FileStream.Free; end; + FillChar(ReadBack, SizeOf(ReadBack), 0); + FileStream := TFileStream.Create(Name, fmOpenRead or fmShareDenyWrite); + try + FileStream.Seek(1024, soBeginning); + FileStream.ReadBuffer(ReadBack[1024], 32); + Report.Check('files.seek', CompareMem(@B[1024],@ReadBack[1024],32), 'Native TFileStream seek/read'); + FileStream.Position := 0; FileStream.ReadBuffer(ReadBack,SizeOf(ReadBack)); + Report.Check('files.native_utf8', CompareMem(@B,@ReadBack,SizeOf(B)), 'UTF-8 path / native filename round trip'); + finally FileStream.Free; end; + Name := WorkDir + 'CaseProbe.txt'; + FileStream := TFileStream.Create(Name,fmCreate); FileStream.Free; + Report.Check('files.case_sensitive', not FileExists(WorkDir+'caseprobe.txt'), 'Android case-sensitive lookup; asset names must match'); + except on E:Exception do Report.Add('files.native','FAIL',E.ClassName+': '+E.Message); end; + try + Raw := TMemoryStream.Create; Compressed := TMemoryStream.Create; + try + C := TCompressionStream.Create(clDefault, Compressed); + try C.WriteBuffer(B,SizeOf(B)); finally C.Free; end; + Compressed.Position := 0; + D := TDecompressionStream.Create(Compressed); + try + repeat + N := D.Read(ReadBack,SizeOf(ReadBack)); + if N>0 then Raw.WriteBuffer(ReadBack,N); + until N=0; + finally D.Free; end; + Report.Check('fpc.zstream', (Raw.Size=SizeOf(B)) and CompareMem(Raw.Memory,@B,SizeOf(B)), + 'Real FPC zlib stream round trip; not a KaM resource decoder test'); + finally Raw.Free; Compressed.Free; end; + except on E:Exception do Report.Add('fpc.zstream','FAIL',E.ClassName+': '+E.Message); end; + try + W := TSumThread.Create(True); + try + W.FreeOnTerminate := False; W.Start; W.WaitFor; + Report.Check('fpc.thread',W.Sum=50005000,'Native worker thread joined; no GUI/simulation callbacks'); + finally W.Free; end; + except on E:Exception do Report.Add('fpc.thread','FAIL',E.ClassName+': '+E.Message); end; + try + Counter := TStringList.Create; + try + Name := WorkDir + 'launch-counter.txt'; Launch := 1; + if FileExists(Name) then begin + Counter.LoadFromFile(Name); + Launch := StrToInt(Trim(Counter.Text))+1; + end; + Counter.Text := IntToStr(Launch); Counter.SaveToFile(Name); + S := 'launch_count='+IntToStr(Launch)+'; compare after force-stop/relaunch'; + if Launch>1 then Report.Add('files.persisted_counter','OBSERVED',S) + else Report.Add('files.persisted_counter','NOT_RUN',S); + finally Counter.Free; end; + except on E:Exception do Report.Add('files.persisted_counter','FAIL',E.ClassName+': '+E.Message); end; +end; +procedure WriteTestTone(const FileName: string); +const SampleRate=22050; Samples=5512; +var F:TFileStream; I:Integer; V:SmallInt; + procedure U16(Value:Word); + var B:array[0..1] of Byte; + begin B[0]:=Value and 255; B[1]:=Value shr 8; F.WriteBuffer(B,2); end; + procedure U32(Value:Cardinal); + var B:array[0..3] of Byte; + begin B[0]:=Value and 255; B[1]:=(Value shr 8) and 255; + B[2]:=(Value shr 16) and 255; B[3]:=Value shr 24; F.WriteBuffer(B,4); end; + procedure FourCC(const S:AnsiString); + begin F.WriteBuffer(S[1],4); end; +begin + F:=TFileStream.Create(FileName,fmCreate); + try + FourCC('RIFF');U32(36+Samples*2);FourCC('WAVE');FourCC('fmt '); + U32(16);U16(1);U16(1);U32(SampleRate);U32(SampleRate*2);U16(2);U16(16); + FourCC('data');U32(Samples*2); + for I:=0 to Samples-1 do begin + V:=Round(3000*Sin(2*Pi*440*I/SampleRate)); + U16(Word(Integer(V) and $FFFF)); + end; + finally F.Free; end; +end; +end. diff --git a/code/ProbeReport.pas b/code/ProbeReport.pas new file mode 100755 index 0000000..4d2ad40 --- /dev/null +++ b/code/ProbeReport.pas @@ -0,0 +1,78 @@ +unit ProbeReport; +{$mode objfpc}{$H+}{$codepage utf8} +interface +uses SysUtils, fpjson; +type + TProbeReport = class + private + FRoot: TJSONObject; + FTests: TJSONArray; + public + constructor Create; + destructor Destroy; override; + procedure Add(const Id, Status, Detail: string); + procedure Check(const Id: string; const Ok: Boolean; const Detail: string); + procedure Save(const FileName: string); + function Status: string; + property Root: TJSONObject read FRoot; + end; +implementation +uses Classes, CastleLog, ProbeBuild; +constructor TProbeReport.Create; +begin + inherited; + FRoot := TJSONObject.Create; + FRoot.Add('schema', 'kam-android-runtime/1'); + FRoot.Add('build_id', ProbeBuildId); + FRoot.Add('profile', ProbeProfile); + FRoot.Add('cpu_expected', ProbeCpu); + FRoot.Add('upstream_commit', ProbeUpstreamCommit); + FRoot.Add('scope', 'Android architecture smoke, not a running KaM world'); + FRoot.Add('world_simulation', 'NOT_RUN'); + FTests := TJSONArray.Create; + FRoot.Add('tests', FTests); +end; +destructor TProbeReport.Destroy; +begin + FRoot.Free; + inherited; +end; +procedure TProbeReport.Add(const Id, Status, Detail: string); +var J: TJSONObject; I: Integer; +begin + for I := FTests.Count - 1 downto 0 do + if FTests.Objects[I].Get('id', '') = Id then FTests.Delete(I); + J := TJSONObject.Create; + J.Add('id', Id); J.Add('status', Status); J.Add('detail', Detail); + FTests.Add(J); + WritelnLog('KaMProbe', Id + ': ' + Status + ' ' + Detail); +end; +procedure TProbeReport.Check(const Id: string; const Ok: Boolean; const Detail: string); +begin + if Ok then Add(Id, 'PASS', Detail) else Add(Id, 'FAIL', Detail); +end; +function TProbeReport.Status: string; +var I: Integer; +begin + Result := 'PASS'; + for I := 0 to FTests.Count - 1 do + if FTests.Objects[I].Get('status','') = 'FAIL' then Exit('FAIL'); + { Optional NOT_RUN / OBSERVED rows do not make the automatic smoke fail. + They remain visible; PASS applies only to executed automatic checks. } +end; +procedure TProbeReport.Save(const FileName: string); +var S: TStringList; +begin + FRoot.Delete('automatic_smoke_status'); + FRoot.Add('automatic_smoke_status', Status); + if not ForceDirectories(ExtractFileDir(FileName)) then + raise Exception.Create('Cannot create report directory'); + S := TStringList.Create; + try + S.Text := FRoot.FormatJSON; + { Evidence must not depend on the filesystem behaviour measured separately + by decision.persistence.rename_replace. } + S.SaveToFile(FileName); + finally S.Free; end; +end; +end. diff --git a/code/ProbeScheduling.pas b/code/ProbeScheduling.pas new file mode 100755 index 0000000..06819e4 --- /dev/null +++ b/code/ProbeScheduling.pas @@ -0,0 +1,89 @@ +unit ProbeScheduling; +{$mode objfpc}{$H+} +{$ifndef ANDROID}{$fatal Decision scheduling probe is Android-only}{$endif} +interface +uses Classes,ProbeReport; +type + TDecisionSchedule=class + private + FReport:TProbeReport; + FWorker:TThread; + FCallbackSteps:Integer; + FCallbackReported,FWorkerReported:Boolean; + FPauses,FResumes:Integer; + public + constructor Create(AReport:TProbeReport); + destructor Destroy;override; + procedure Tick; + procedure Paused; + procedure Resumed; + procedure ObserveInstanceCount; + end; +implementation +uses SysUtils; +type + TDecisionWorker=class(TThread) + public + Executions:Integer; + Sum:Int64; + Done:Boolean; + procedure Execute;override; + end; +var GScheduleInstances:Integer=0; +procedure TDecisionWorker.Execute; +var I:Integer; +begin + Inc(Executions);Sum:=0; + try + for I:=1 to 40 do begin + if Terminated then Exit; + Sum:=Sum+I;Sleep(100); + end; + finally Done:=True;end; +end; +constructor TDecisionSchedule.Create(AReport:TProbeReport); +begin + inherited Create;FReport:=AReport;Inc(GScheduleInstances); + FWorker:=TDecisionWorker.Create(True);FWorker.FreeOnTerminate:=False;FWorker.Start; + ObserveInstanceCount; +end; +destructor TDecisionSchedule.Destroy; +begin + if FWorker<>nil then begin + FWorker.Terminate;FWorker.WaitFor;FWorker.Free; + end; + inherited; +end; +procedure TDecisionSchedule.ObserveInstanceCount; +begin + FReport.Add('decision.schedule.host_instance_count','OBSERVED','instances_in_process='+IntToStr(GScheduleInstances)); +end; +procedure TDecisionSchedule.Tick; +var W:TDecisionWorker; +begin + if not FCallbackReported then begin + Inc(FCallbackSteps); + if FCallbackSteps>=120 then begin + FCallbackReported:=True; + FReport.Add('decision.schedule.callback_progress','OBSERVED', + 'completed=true; steps='+IntToStr(FCallbackSteps)+'; pauses='+IntToStr(FPauses)+'; resumes='+IntToStr(FResumes)); + end; + end; + if not FWorkerReported then begin + W:=TDecisionWorker(FWorker); + if W.Done then begin + W.WaitFor;FWorkerReported:=True; + FReport.Add('decision.schedule.worker_progress','OBSERVED', + 'completed='+LowerCase(BoolToStr((W.Executions=1) and (W.Sum=820),True))+ + '; executions='+IntToStr(W.Executions)+'; sum='+IntToStr(W.Sum)+ + '; pauses='+IntToStr(FPauses)+'; resumes='+IntToStr(FResumes)); + end; + end; +end; +procedure TDecisionSchedule.Paused; +begin Inc(FPauses);FReport.Add('decision.schedule.lifecycle','OBSERVED', + 'event=pause; pauses='+IntToStr(FPauses)+'; resumes='+IntToStr(FResumes));end; +procedure TDecisionSchedule.Resumed; +begin Inc(FResumes);ObserveInstanceCount;FReport.Add('decision.schedule.lifecycle','OBSERVED', + 'event=resume; pauses='+IntToStr(FPauses)+'; resumes='+IntToStr(FResumes));end; +end. diff --git a/code/ProbeTextProvider.pas b/code/ProbeTextProvider.pas new file mode 100755 index 0000000..66b4c29 --- /dev/null +++ b/code/ProbeTextProvider.pas @@ -0,0 +1,34 @@ +unit ProbeTextProvider; +{$mode objfpc}{$H+} +interface +uses ProbeReport; +procedure RunTextProviderChecks(Report:TProbeReport); +implementation +uses SysUtils; +function ByteEqual(const S:RawByteString;const B:array of Byte):Boolean; +var I:Integer; +begin + Result:=Length(S)=Length(B);if not Result then Exit; + for I:=0 to High(B) do if Byte(S[I+1])<>B[I] then Exit(False); +end; +procedure RunTextProviderChecks(Report:TProbeReport); +const CP1251Bytes:array[0..5] of Byte=($CF,$F0,$E8,$E2,$E5,$F2); +var A,Back:RawByteString;U:UnicodeString;I:Integer;OkDecode,OkEncode:Boolean; +begin + try + SetLength(A,Length(CP1251Bytes));for I:=0 to High(CP1251Bytes) do A[I+1]:=AnsiChar(CP1251Bytes[I]); + SetCodePage(A,1251,False);U:=UnicodeString(A); + OkDecode:=(Length(U)=6) and (Ord(U[1])=$041F) and (Ord(U[2])=$0440) and + (Ord(U[3])=$0438) and (Ord(U[4])=$0432) and (Ord(U[5])=$0435) and (Ord(U[6])=$0442); + Report.Add('decision.text.cp1251_decode','OBSERVED', + 'supported='+LowerCase(BoolToStr(OkDecode,True))+'; source_codepage='+IntToStr(StringCodePage(A))); + Back:=UTF8Encode(U);SetCodePage(Back,1251,True);OkEncode:=ByteEqual(Back,CP1251Bytes); + Report.Add('decision.text.cp1251_encode','OBSERVED', + 'supported='+LowerCase(BoolToStr(OkEncode,True))+'; result_codepage='+IntToStr(StringCodePage(Back))); + except on E:Exception do begin + Report.Add('decision.text.cp1251_decode','OBSERVED','supported=false; error='+E.ClassName+': '+E.Message); + Report.Add('decision.text.cp1251_encode','OBSERVED','supported=false; error='+E.ClassName+': '+E.Message); + end; +end; +end; +end. diff --git a/config/extra-paths.json b/config/extra-paths.json new file mode 100755 index 0000000..6d5d07b --- /dev/null +++ b/config/extra-paths.json @@ -0,0 +1,4 @@ +{ + "source_paths": [], + "library_paths": [] +} diff --git a/config/profiles.json b/config/profiles.json new file mode 100755 index 0000000..11d4a55 --- /dev/null +++ b/config/profiles.json @@ -0,0 +1,19 @@ +[ + {"id":"platform","units":[],"required":true,"package":true,"purpose":"Castle/FPC Android shell, storage, threads, compression, touch, procedural GPU image, audio API, lifecycle"}, + {"id":"core_types","units":["KM_CommonTypes","KM_Points"],"required":true,"package":true,"purpose":"Original core types, point arithmetic and ABI witness"}, + {"id":"core_streams","units":["KM_CommonClasses"],"required":true,"package":false,"purpose":"Original serialization stream, records, strings and compression dependencies"}, + {"id":"mission_tokens","units":["KM_MissionScript"],"required":true,"package":false,"purpose":"Original DAT tokenizer and its real transitive dependencies"}, + {"id":"mission_info","units":["KM_MissionScript_Info"],"required":true,"package":false,"purpose":"Original MAP/DAT metadata loader dependency and native-link boundary; no world execution"}, + {"id":"terrain_io","units":["KM_TerrainUtils"],"required":true,"package":false,"purpose":"Original terrain binary I/O dependency boundary"}, + {"id":"sprite_loader","units":["KM_ResSprites"],"required":true,"package":false,"purpose":"Original RX/RXX loader and its graphics/palette/compression dependencies"}, + {"id":"terrain_pathfinding","units":["KM_Terrain"],"required":true,"package":false,"purpose":"Original terrain/pathfinding dependency graph"}, + {"id":"mission_world","units":["KM_MissionScript_Standard"],"required":true,"package":false,"purpose":"Original full mission loader, buildings/units command dependencies"}, + {"id":"simulation","units":["KM_Game"],"required":true,"package":false,"purpose":"Original game core dependency graph, not the desktop executable"}, + {"id":"script_vm","units":["uPSCompiler","uPSRuntime"],"required":true,"package":false,"purpose":"Pinned PascalScript submodule source and ARM ABI support"}, + {"id":"scripting","units":["KM_ScriptingEvents"],"required":true,"package":false,"purpose":"KaM scripting integration dependency graph"}, + {"id":"legacy_render","units":["KM_Render"],"required":false,"package":false,"purpose":"Diagnostic only: discover original renderer's platform dependencies"}, + {"id":"legacy_audio","units":["KM_Sound"],"required":false,"package":false,"purpose":"Diagnostic only: discover original audio's platform dependencies"}, + {"id":"integration","units":["KM_CommonTypes","KM_Points","KM_CommonClasses","KM_MissionScript_Info","KM_TerrainUtils","KM_ResSprites","KM_Terrain","KM_MissionScript_Standard","KM_Game","KM_ScriptingEvents"],"required":true,"package":true,"purpose":"Link actual core roots together with Castle in ONE Android native library and APK"}, + {"id":"oq_host_view","units":[],"required":false,"package":true,"game_unit":"DecisionHostView","purpose":"Decision evidence for T1-RP01-Q01: minimal TCastleView host without child controls"}, + {"id":"oq_host_callback","units":[],"required":false,"package":true,"game_unit":"DecisionHostCallback","purpose":"Decision evidence for T1-RP01-Q01: Application.OnInitialize host without a custom view"} +] diff --git a/config/upstream.json b/config/upstream.json new file mode 100755 index 0000000..28355f1 --- /dev/null +++ b/config/upstream.json @@ -0,0 +1,7 @@ +{ + "repository": "reyandme/kam_remake", + "tag": "r16020", + "commit": "962fb850324b79bbe5480e1c2087d98f83a60917", + "submodules": {"src/ext/pascalscript_submodule": "reyandme/pascalscript"}, + "source_only": true +} diff --git a/open_questions/config/fixture_manifest.json b/open_questions/config/fixture_manifest.json new file mode 100755 index 0000000..df9e2e3 --- /dev/null +++ b/open_questions/config/fixture_manifest.json @@ -0,0 +1,57 @@ +{ + "schema": "kam-task1-open-question-fixtures-v2", + "selection_policy": "exact-whitelist-no-recursive-discovery", + "asset_root_relative": "private-assets", + "primary": [ + { + "id": "town-defence-dat", + "role": "dat", + "path": "Maps/Town Defence/Town Defence.dat", + "expected_size": 5350, + "logical_fixture": "town-defence-pair" + }, + { + "id": "town-defence-map", + "role": "map", + "path": "Maps/Town Defence/Town Defence.map", + "expected_size": 281646, + "logical_fixture": "town-defence-pair" + }, + { + "id": "tileset-rxx", + "role": "rxx", + "path": "data/sprites/Tileset.rxx", + "expected_size": 447964, + "resource_category": "rxTiles", + "logical_fixture": "tileset-rxx" + } + ], + "conditional_fallbacks": [ + { + "id": "layer-map", + "enabled_by_default": false, + "condition": "Use only if Town Defence.map does not exercise the layered/generated-terrain branch required by T1-RP03-Q02/T1-RP03-Q05.", + "files": [ + { + "id": "tbk01-dat", + "role": "dat", + "path": "Maps/The Barbarian King/TBK01/TBK01.dat", + "expected_size": 52903, + "logical_fixture": "tbk01-pair" + }, + { + "id": "tbk01-map", + "role": "map", + "path": "Maps/The Barbarian King/TBK01/TBK01.map", + "expected_size": 368686, + "logical_fixture": "tbk01-pair" + } + ] + } + ], + "notes": [ + "Paths are relative to the existing project private-assets directory.", + "The probe must never recursively discover binary fixtures.", + "Do not add GUI.rxa, Units.rxx, Houses.rxx or any other resource pack unless a future planning decision explicitly changes this manifest." + ] +} diff --git a/open_questions/config/questions.json b/open_questions/config/questions.json new file mode 100755 index 0000000..98c7e9c --- /dev/null +++ b/open_questions/config/questions.json @@ -0,0 +1,126 @@ +{ + "schema": "kam-task1-oq-probe/1", + "questions": [ + { + "id": "T1-Q01", + "tests": [ + "fixture_inventory", + "storage_contract" + ] + }, + { + "id": "T1-Q02", + "tests": [ + "expected_control_independence" + ] + }, + { + "id": "T1-RP01-Q01", + "tests": [ + "castle_host_variants" + ] + }, + { + "id": "T1-RP01-Q02", + "tests": [ + "bounded_workload", + "lifecycle_interrupt" + ] + }, + { + "id": "T1-RP01-Q03", + "tests": [ + "evidence_crash_matrix", + "android_atomic_replace" + ] + }, + { + "id": "T1-RP02-Q01", + "tests": [ + "portable_closure_inventory", + "compile_variant_matrix" + ] + }, + { + "id": "T1-RP02-Q02", + "tests": [ + "text_encoding_inventory", + "text_encoding_vectors" + ] + }, + { + "id": "T1-RP02-Q03", + "tests": [ + "stream_read_inventory", + "truncation_matrix" + ] + }, + { + "id": "T1-RP03-Q01", + "tests": [ + "dat_command_inventory", + "dat_projection_compare" + ] + }, + { + "id": "T1-RP03-Q02", + "tests": [ + "terrain_generation_dependency", + "layered_map_probe" + ] + }, + { + "id": "T1-RP03-Q03", + "tests": [ + "map_memory_estimate", + "map_projection_equivalence" + ] + }, + { + "id": "T1-RP03-Q04", + "tests": [ + "map_revision_inventory", + "map_tail_truncation" + ] + }, + { + "id": "T1-RP03-Q05", + "tests": [ + "fixture_branch_coverage" + ] + }, + { + "id": "T1-RP04-Q01", + "tests": [ + "rxx_fixture_inventory", + "rxx_branch_coverage" + ] + }, + { + "id": "T1-RP04-Q02", + "tests": [ + "sprite_cpu_closure", + "sprite_compile_variants" + ] + }, + { + "id": "T1-RP04-Q03", + "tests": [ + "sprite_memory_estimate", + "sprite_projection_equivalence" + ] + }, + { + "id": "T1-RP05-Q01", + "tests": [ + "failure_isolation" + ] + }, + { + "id": "T1-RP05-Q02", + "tests": [ + "repeatability_matrix" + ] + } + ] +} diff --git a/open_questions/tests/test_probe.py b/open_questions/tests/test_probe.py new file mode 100755 index 0000000..2c89ed0 --- /dev/null +++ b/open_questions/tests/test_probe.py @@ -0,0 +1,106 @@ +import importlib.util +import json +import tempfile +import unittest +from pathlib import Path + +HERE = Path(__file__).resolve().parents[1] +PROJECT = HERE.parent + + +def load_probe_module(): + p = HERE / 'tools/decision_probe.py' + spec = importlib.util.spec_from_file_location('decision_probe_testmod', p) + mod = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(mod) + return mod + + +class ProbeTests(unittest.TestCase): + def test_question_count_and_unique(self): + q = json.loads((HERE/'config/questions.json').read_text(encoding='utf-8'))['questions'] + ids = [x['id'] for x in q] + self.assertEqual(18, len(ids)) + self.assertEqual(len(ids), len(set(ids))) + + def test_every_question_has_test(self): + q = json.loads((HERE/'config/questions.json').read_text(encoding='utf-8'))['questions'] + self.assertTrue(all(x['tests'] for x in q)) + + def test_fixed_manifest_is_minimal_and_exact(self): + m = json.loads((HERE/'config/fixture_manifest.json').read_text(encoding='utf-8')) + self.assertEqual('exact-whitelist-no-recursive-discovery', m['selection_policy']) + self.assertEqual([ + 'Maps/Town Defence/Town Defence.dat', + 'Maps/Town Defence/Town Defence.map', + 'data/sprites/Tileset.rxx', + ], [x['path'] for x in m['primary']]) + self.assertEqual(1, len(m['conditional_fallbacks'])) + self.assertFalse(m['conditional_fallbacks'][0]['enabled_by_default']) + self.assertEqual([ + 'Maps/The Barbarian King/TBK01/TBK01.dat', + 'Maps/The Barbarian King/TBK01/TBK01.map', + ], [x['path'] for x in m['conditional_fallbacks'][0]['files']]) + + def test_binary_selection_ignores_decoys_and_fallback_is_opt_in(self): + mod = load_probe_module() + with tempfile.TemporaryDirectory() as td: + root = Path(td) + assets = root/'private-assets' + (assets/'a').mkdir(parents=True) + (assets/'decoys').mkdir(parents=True) + (assets/'a/one.dat').write_bytes(b'abc') + (assets/'a/two.map').write_bytes(b'defg') + (assets/'a/three.rxx').write_bytes(b'hijkl') + (assets/'a/fallback.map').write_bytes(b'mnopqr') + (assets/'decoys/should-never-be-read.map').write_bytes(b'X'*100) + manifest = root/'manifest.json' + manifest.write_text(json.dumps({ + 'selection_policy':'exact-whitelist-no-recursive-discovery', + 'primary':[ + {'id':'d','role':'dat','path':'a/one.dat','expected_size':3}, + {'id':'m','role':'map','path':'a/two.map','expected_size':4}, + {'id':'r','role':'rxx','path':'a/three.rxx','expected_size':5}, + ], + 'conditional_fallbacks':[ + {'id':'layer-map','enabled_by_default':False,'files':[ + {'id':'fm','role':'map','path':'a/fallback.map','expected_size':6} + ]} + ] + }), encoding='utf-8') + mod.ASSETS = assets + mod.MANIFEST = manifest + mod.INCLUDE_FALLBACKS = set() + primary = [p.relative_to(assets).as_posix() for _,p in mod.selected_files({'.dat','.map','.rxx'})] + self.assertEqual(['a/one.dat','a/two.map','a/three.rxx'], primary) + self.assertNotIn('decoys/should-never-be-read.map', primary) + mod.INCLUDE_FALLBACKS = {'layer-map'} + with_fallback = [p.relative_to(assets).as_posix() for _,p in mod.selected_files({'.dat','.map','.rxx'})] + self.assertEqual(['a/one.dat','a/two.map','a/three.rxx','a/fallback.map'], with_fallback) + + def test_independent_inspector_does_not_import_kam(self): + s = (HERE/'tools/decision_probe.py').read_text(encoding='utf-8') + self.assertNotIn('import KM_', s) + self.assertNotIn('from KM_', s) + + def test_no_recursive_binary_asset_scan_in_probe(self): + s = (HERE/'tools/decision_probe.py').read_text(encoding='utf-8') + self.assertNotIn('ASSETS.rglob', s) + self.assertNotIn('ASSETS.glob', s) + self.assertNotIn("rglob('*')", s) + self.assertNotIn("glob('**", s) + + def test_integrated_paths_do_not_replace_old_runner(self): + self.assertTrue((PROJECT/'run-open-question-tests.sh').is_file()) + compatibility = (HERE/'COMPATIBILITY.md').read_text(encoding='utf-8') + self.assertIn('build-android.sh', compatibility) + self.assertIn('does not overwrite', compatibility) + + def test_no_production_claim(self): + s = (HERE/'README.md').read_text(encoding='utf-8') + self.assertIn('does not replace', s) + + +if __name__ == '__main__': + unittest.main() diff --git a/open_questions/tools/decision_probe.py b/open_questions/tools/decision_probe.py new file mode 100755 index 0000000..1b03625 --- /dev/null +++ b/open_questions/tools/decision_probe.py @@ -0,0 +1,272 @@ +#!/usr/bin/env python3 +from __future__ import annotations +import argparse, hashlib, json, re, struct +from pathlib import Path + +HERE = Path(__file__).resolve().parents[1] +PROJECT_ROOT = HERE.parent +SRC = PROJECT_ROOT / 'vendor/kam_remake' +ASSETS = PROJECT_ROOT / 'private-assets' +MANIFEST = HERE / 'config/fixture_manifest.json' +OUT = PROJECT_ROOT / 'reports/open-questions/current' +INCLUDE_FALLBACKS: set[str] = set() + + +def sha(p: Path) -> str: + return hashlib.sha256(p.read_bytes()).hexdigest() + + +def text(p: Path) -> str: + return p.read_text(encoding='utf-8-sig', errors='replace') + + +def emit(name: str, data: dict) -> dict: + OUT.mkdir(parents=True, exist_ok=True) + (OUT / (name + '.json')).write_text(json.dumps(data, indent=2, ensure_ascii=False) + '\n', encoding='utf-8') + return data + + +def load_manifest() -> dict: + return json.loads(MANIFEST.read_text(encoding='utf-8')) + + +def selected_specs() -> list[dict]: + m = load_manifest() + specs = [dict(x, selection='primary') for x in m.get('primary', [])] + known = {x['id'] for x in m.get('conditional_fallbacks', [])} + unknown = INCLUDE_FALLBACKS - known + if unknown: + raise SystemExit('Unknown fallback id(s): ' + ', '.join(sorted(unknown))) + for group in m.get('conditional_fallbacks', []): + if group['id'] in INCLUDE_FALLBACKS: + specs.extend(dict(x, selection='fallback:' + group['id']) for x in group.get('files', [])) + return specs + + +def resolve_spec(spec: dict) -> Path: + rel = Path(spec['path']) + if rel.is_absolute() or '..' in rel.parts: + raise SystemExit(f"Unsafe fixture path in manifest: {spec['path']}") + root = ASSETS.resolve() + p = (root / rel).resolve() + try: + p.relative_to(root) + except ValueError: + raise SystemExit(f"Fixture escaped asset root: {spec['path']}") + return p + + +def selected_files(exts: set[str]) -> list[tuple[dict, Path]]: + rows = [] + for spec in selected_specs(): + p = resolve_spec(spec) + if p.suffix.lower() in exts: + rows.append((spec, p)) + return rows + + +def fixture_manifest_status() -> dict: + rows = [] + good = True + for spec in selected_specs(): + p = resolve_spec(spec) + row = { + 'id': spec['id'], + 'role': spec['role'], + 'path': spec['path'], + 'selection': spec['selection'], + 'expected_size': spec.get('expected_size'), + 'exists': p.is_file(), + } + if p.is_file(): + row['actual_size'] = p.stat().st_size + row['size_matches'] = spec.get('expected_size') in (None, p.stat().st_size) + if not row['size_matches']: + good = False + else: + row['actual_size'] = None + row['size_matches'] = False + good = False + rows.append(row) + return emit('fixture_manifest_status', { + 'test': 'fixture_manifest_status', + 'selection_policy': load_manifest().get('selection_policy'), + 'assets_root': str(ASSETS), + 'manifest': str(MANIFEST), + 'included_fallbacks': sorted(INCLUDE_FALLBACKS), + 'fixtures': rows, + 'status': 'PASS' if good else 'MISSING_OR_MISMATCH' + }) + + +def fixture_inventory(): + rows = [] + status = 'PASS' + for spec, p in selected_files({'.map', '.dat', '.rxx', '.rx', '.rxa', '.txt', '.libx', '.script'}): + if not p.is_file(): + rows.append({'id': spec['id'], 'path': spec['path'], 'role': spec['role'], 'status': 'MISSING'}) + status = 'MISSING_OR_MISMATCH' + continue + size = p.stat().st_size + size_ok = spec.get('expected_size') in (None, size) + if not size_ok: + status = 'MISSING_OR_MISMATCH' + rows.append({ + 'id': spec['id'], + 'path': spec['path'], + 'role': spec['role'], + 'logical_fixture': spec.get('logical_fixture'), + 'selection': spec['selection'], + 'size': size, + 'expected_size': spec.get('expected_size'), + 'size_matches': size_ok, + 'sha256': sha(p), + 'ext': p.suffix.lower(), + 'resource_category': spec.get('resource_category') + }) + return emit('fixture_inventory', { + 'test': 'fixture_inventory', + 'count': len(rows), + 'fixtures': rows, + 'status': status if rows else 'NEEDS_FIXTURES', + 'selection_policy': 'exact manifest whitelist; no directory scan' + }) + + +def source_patterns(): + pats = { + 'desktop_units': r'\b(?:LCLIntf|LCLType|Graphics|FileUtil|LazUTF8|LazFileUtils|lconvencoding)\b', + 'read_calls': r'\.Read(?:Buffer)?\s*\(', + 'terrain_generation': r'GenerateTerrainTransitions|GetGenTerrainInfoLegacy|GetGenTerrainInfo', + 'tokenizer': r'TokenizeScript\s*\(', + 'render_globals': r'gGFXData|SKIP_RENDER|NO_OGL' + } + out = {k: [] for k in pats} + if SRC.exists(): + for p in SRC.rglob('*.pas'): + s = text(p) + for k, pat in pats.items(): + for m in re.finditer(pat, s, re.I): + out[k].append({'file': p.relative_to(SRC).as_posix(), + 'line': s.count('\n', 0, m.start()) + 1, 'match': m.group(0)}) + return emit('source_inventory', {'test': 'portable_closure_inventory', + 'status': 'PASS' if (SRC/'SOURCE_RECEIPT.json').exists() else 'BLOCKED', + 'patterns': out}) + + +def dat_scan(): + p = SRC/'src/mission/KM_MissionScript.pas' + cmd = re.findall(r"'([A-Z][A-Z0-9_]*)'", text(p)) if p.exists() else [] + rows = [] + for spec, f in selected_files({'.dat'}): + if not f.is_file(): + continue + b = f.read_bytes() + printable = sum(c in b'\t\n\r 0123456789!' for c in b) / max(1, len(b)) + rows.append({'id': spec['id'], 'path': spec['path'], 'size': len(b), + 'likely_encoded': printable < .20, 'bang_count_raw': b.count(b'!')}) + return emit('dat_inventory', {'test': 'dat_command_inventory', 'recognized_source_tokens': sorted(set(cmd)), + 'fixtures': rows, 'status': 'PASS' if p.exists() and rows else 'BLOCKED'}) + + +def map_scan(): + rows = [] + for spec, f in selected_files({'.map'}): + if not f.is_file(): + continue + b = f.read_bytes() + r = {'id': spec['id'], 'path': spec['path'], 'size': len(b), 'sha256': hashlib.sha256(b).hexdigest()} + if len(b) >= 8: + x = struct.unpack_from(' int: + global SRC, ASSETS, MANIFEST, OUT, INCLUDE_FALLBACKS + ap = argparse.ArgumentParser(description='Task 1 open-question evidence selector with an exact binary-fixture whitelist.') + ap.add_argument('action', choices=['verify', 'all', 'source']) + ap.add_argument('--source', type=Path, default=SRC) + ap.add_argument('--assets-root', type=Path, default=ASSETS) + ap.add_argument('--fixture-manifest', type=Path, default=MANIFEST) + ap.add_argument('--include-fallback', action='append', default=[]) + ap.add_argument('--out', type=Path, default=OUT) + a = ap.parse_args() + SRC = a.source.resolve(); ASSETS = a.assets_root.resolve(); MANIFEST = a.fixture_manifest.resolve(); OUT = a.out.resolve() + INCLUDE_FALLBACKS = set(a.include_fallback) + if a.action == 'verify': + result = fixture_manifest_status() + return 0 if result['status'] == 'PASS' else 2 + if a.action == 'all': + allrun() + else: + source_patterns() + return 0 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/run-open-question-tests.sh b/run-open-question-tests.sh new file mode 100755 index 0000000..96bc217 --- /dev/null +++ b/run-open-question-tests.sh @@ -0,0 +1,102 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +cd "$ROOT" + +SOURCE="$ROOT/vendor/kam_remake" +ASSETS="$ROOT/private-assets" +FIXTURE_MANIFEST="$ROOT/open_questions/config/fixture_manifest.json" +INCLUDE_FALLBACK=() + +usage() { + cat < private-assets/Maps/The Barbarian King/TBK01/TBK01.dat + TBK01.map + +No recursive fixture discovery is performed. +This script does NOT run build-android.sh and does NOT execute Task 1 implementation. +EOF2 +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --source) SOURCE="${2:?missing path after --source}"; shift 2 ;; + --assets-root) ASSETS="${2:?missing path after --assets-root}"; shift 2 ;; + --include-layer-fallback) INCLUDE_FALLBACK=(--include-fallback layer-map); shift ;; + -h|--help) usage; exit 0 ;; + *) echo "Unknown argument: $1" >&2; usage >&2; exit 2 ;; + esac +done + +if ! python3 tools/source_fetch.py --verify --destination "$SOURCE" >/dev/null 2>&1; then + if [[ "$SOURCE" != "$ROOT/vendor/kam_remake" ]]; then + echo "Pinned source at $SOURCE is missing or invalid." >&2 + exit 2 + fi + echo "Pinned r16020 source is absent; fetching it with the EXISTING project source_fetch.py..." + python3 tools/source_fetch.py --destination "$SOURCE" +fi + +RUN_ID="$(date -u +%Y%m%dT%H%M%SZ)" +OUT="$ROOT/reports/open-questions/runs/$RUN_ID" +mkdir -p "$OUT" + +# Fail before any binary probe if an exact whitelisted path is absent or has an unexpected size. +python3 open_questions/tools/decision_probe.py verify \ + --source "$SOURCE" \ + --assets-root "$ASSETS" \ + --fixture-manifest "$FIXTURE_MANIFEST" \ + "${INCLUDE_FALLBACK[@]}" \ + --out "$OUT" + +python3 open_questions/tools/decision_probe.py all \ + --source "$SOURCE" \ + --assets-root "$ASSETS" \ + --fixture-manifest "$FIXTURE_MANIFEST" \ + "${INCLUDE_FALLBACK[@]}" \ + --out "$OUT" +python3 -m unittest discover -s open_questions/tests -v + +python3 - "$OUT" "$ROOT/reports/open-questions" <<'PY' +import json,sys +from pathlib import Path +out=Path(sys.argv[1]); root=Path(sys.argv[2]) +idx=json.loads((out/'decision_index.json').read_text(encoding='utf-8')) +inv=json.loads((out/'fixture_inventory.json').read_text(encoding='utf-8')) +branch=json.loads((out/'fixture_branch_coverage.json').read_text(encoding='utf-8')) +manifest=json.loads((out/'fixture_manifest_status.json').read_text(encoding='utf-8')) +latest={'run':out.name,'directory':str(out),'decision_index':str(out/'decision_index.json'), + 'fixture_count':inv['count'],'fixture_status':inv['status'],'branch_status':branch['status'], + 'manifest_status':manifest['status'],'included_fallbacks':idx.get('included_fallbacks',[])} +root.mkdir(parents=True,exist_ok=True) +(root/'LATEST.json').write_text(json.dumps(latest,indent=2,ensure_ascii=False)+'\n',encoding='utf-8') +summary = [ + '# Task 1 open-question probe — latest run','', + f"Run: `{out.name}`", f"Whitelisted files read: **{inv['count']}** ({inv['status']})", + f"Exact manifest paths/sizes: **{manifest['status']}**", + f"Required format families: **{branch['status']}**", + f"Conditional fallbacks enabled: **{', '.join(idx.get('included_fallbacks',[])) or 'none'}**",'', + 'No recursive binary fixture discovery is used.', + 'This run contains only static/fixture evidence. Android runtime micro-tests are a separate stage.', + '',f"Decision index: `{out/'decision_index.json'}`" +] +(root/'LATEST-SUMMARY.md').write_text('\n'.join(summary)+'\n',encoding='utf-8') +PY + +echo +echo "Open-question static probe completed." +echo "Summary: reports/open-questions/LATEST-SUMMARY.md" +echo "Index: reports/open-questions/LATEST.json" +echo "Binary fixture selection came ONLY from open_questions/config/fixture_manifest.json." +echo "This command did NOT run build-android.sh." diff --git a/test-harness.sh b/test-harness.sh new file mode 100755 index 0000000..0a54e80 --- /dev/null +++ b/test-harness.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +# Tests Python orchestration on the build server. DOES NOT compile a desktop game. +set -euo pipefail +ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +cd "$ROOT" +exec python3 -m unittest discover -s tests -v diff --git a/tests/helpers.py b/tests/helpers.py new file mode 100755 index 0000000..5091b92 --- /dev/null +++ b/tests/helpers.py @@ -0,0 +1,48 @@ +"""Generated bytes below are TEST DOUBLES, never real compiler/APK evidence.""" +import struct + +def elf(cpu='aarch64',needed=(),textrel=False): + cls=1 if cpu=='arm' else 2 + machine={'arm':40,'aarch64':183,'x86_64':62}[cpu] + strings=b'\0';offsets=[] + for name in needed:offsets.append(len(strings));strings+=name.encode()+b'\0' + data=bytearray(1024);data[:7]=b'\x7fELF'+bytes([cls,1,1]) + struct.pack_into('dict: + abi,machine,elfclass,_=CPUS[cpu] + if len(data)<64 or data[:4]!=b'\x7fELF': raise ProbeError('Not an ELF library') + if data[4]!=elfclass or data[5]!=1: raise ProbeError('Wrong ELF class or byte order') + kind,actual=struct.unpack_from('len(data): + raise ProbeError('Invalid ELF program header table') + loads=[];dynamic=None + for i in range(phnum): + pos=phoff+i*phsize + if elfclass==2: + typ,flags,offset,vaddr,_,filesz,memsz,align=struct.unpack_from('len(data):raise ProbeError('ELF segment outside file') + if typ==1:loads.append({'offset':offset,'vaddr':vaddr,'size':filesz,'alignment':align}) + if typ==2:dynamic=(offset,filesz) + if not loads or dynamic is None:raise ProbeError('Missing ELF load/dynamic segments') + tags=[];fmt='len(data):raise ProbeError('Invalid string table size') + for off in needed_offsets: + if off>=stringsize:raise ProbeError('Invalid DT_NEEDED index') + end=data.find(b'\0',start+off,start+stringsize) + if end<0:raise ProbeError('Unterminated DT_NEEDED') + needed.append(data[start+off:end].decode('ascii')) + return {'abi':abi,'machine':machine,'elf_class':elfclass,'type':'DYN','needed':needed, + 'load_segments':loads,'elf_16k_alignment_candidate':all(s['alignment']>=16384 and + s['offset']%16384==s['vaddr']%16384 for s in loads), + 'note':'ELF audit only. dlopen dependencies and Android loader execution require device test.'} + +def runtime_identity(data:bytes,cpu:str)->str: + """Fingerprint loadable sections/layout while allowing removal of debug symbols. + + Android packaging may strip non-allocated sections. Do not require the whole + file to be byte-identical, but never accept different code/data or a library + without a usable section table on this fallback path. + """ + inspect_elf(data,cpu) + cls=CPUS[cpu][2] + if cls==2: + entry,phoff,shoff=struct.unpack_from('len(data): + raise ProbeError('Cannot verify stripped library: invalid/absent section table') + sections=[struct.unpack_from(fmt,data,shoff+i*shsize) for i in range(shnum)] + names=sections[shstr]; ns,nn=names[4],names[5] + if ns+nn>len(data):raise ProbeError('Invalid ELF section-name table') + records=[];seen=set() + for name,typ,flags,addr,off,size,link,info,align,entsize in sections: + if not flags&2:continue # SHF_ALLOC; debug/symbol-only data is excluded. + if name>=nn:raise ProbeError('Invalid section-name offset') + end=data.find(b'\0',ns+name,ns+nn) + if end<0:raise ProbeError('Invalid section name') + text=data[ns+name:end].decode('ascii') + key=(text,addr) + if key in seen:raise ProbeError('Duplicate allocated section') + seen.add(key) + if typ!=8 and off+size>len(data):raise ProbeError('Allocated section outside ELF') + # Link/info may be section indices and legitimately change on strip. + records.append([text,typ,flags,addr,size,align,entsize, + None if typ==8 else hashlib.sha256(data[off:off+size]).hexdigest()]) + if not records or not any(r[2]&4 for r in records): + raise ProbeError('Cannot verify stripped library: no executable allocated sections') + shape={'entry':entry,'program_headers':data[phoff:phoff+phsize*phnum].hex(), + 'sections':sorted(records)} + return hashlib.sha256(json.dumps(shape,sort_keys=True).encode()).hexdigest() + +def inspect_apk(path:Path,cpu:str,library_name:str,compiled_sha:str,compiled_data:bytes|None=None)->dict: + abi=CPUS[cpu][0] + with zipfile.ZipFile(path) as z: + if len(z.namelist())!=len(set(z.namelist())):raise ProbeError('Duplicate ZIP entries in APK') + if z.testzip() is not None:raise ProbeError('APK ZIP CRC failure') + for required in ('AndroidManifest.xml','classes.dex'): + if required not in z.namelist():raise ProbeError('APK lacks '+required) + if not z.read('classes.dex').startswith(b'dex\n'):raise ProbeError('APK has invalid DEX magic') + libs={Path(n).name:n for n in z.namelist() if n.startswith('lib/'+abi+'/') and n.endswith('.so')} + if not libs:raise ProbeError('APK lacks native libraries for '+abi) + matches=[] + for candidate,path_in_apk in libs.items(): + data=z.read(path_in_apk);digest=hashlib.sha256(data).hexdigest() + if digest==compiled_sha: + matches.append((candidate,path_in_apk,data,digest,'exact_sha256'));continue + if compiled_data is None or hashlib.sha256(compiled_data).hexdigest()!=compiled_sha:continue + try:same_runtime=runtime_identity(data,cpu)==runtime_identity(compiled_data,cpu) + except ProbeError:continue + if same_runtime:matches.append((candidate,path_in_apk,data,digest,'identical_allocated_sections_and_program_headers_after_stripping')) + if len(matches)!=1:raise ProbeError('APK has no unique native library matching the compiled Android runtime; artifact may be different/stale') + main_name,main_path,main,packaged_sha,identity=matches[0] + records={} + for name,entry in libs.items(): + rec=inspect_elf(z.read(entry),cpu) + missing=sorted(set(rec['needed'])-set(libs)-ANDROID_SYSTEM) + if missing:raise ProbeError('Unpackaged DT_NEEDED for '+name+': '+', '.join(missing)) + info=z.getinfo(entry);rec['zip_compression']=info.compress_type + records[name]=rec + unexpected=sorted({n.split('/')[1] for n in z.namelist() if n.startswith('lib/') and n.endswith('.so')}- {abi}) + if unexpected:raise ProbeError('Unexpected ABI in single-CPU probe: '+', '.join(unexpected)) + return {'sha256':sha256(path),'size':path.stat().st_size,'abi':abi,'native_libraries':records, + 'main_library_path':main_path,'main_library_name':main_name,'main_library_identity':identity,'packaged_main_sha256':packaged_sha, + 'signing':'Verified separately by apksigner when available; ZIP inspection is not signature verification'} diff --git a/tools/common.py b/tools/common.py new file mode 100755 index 0000000..1112694 --- /dev/null +++ b/tools/common.py @@ -0,0 +1,80 @@ +"""Small, dependency-free utilities used by the Android-only probe.""" +from __future__ import annotations +import hashlib, json, os, re, shlex, signal, subprocess, time +from pathlib import Path, PurePosixPath +from datetime import datetime, timezone + +ROOT = Path(__file__).resolve().parents[1] +COMMIT = '962fb850324b79bbe5480e1c2087d98f83a60917' +REPOSITORY = 'reyandme/kam_remake' +VERSION = '0.4.0' +CPUS = {'aarch64': ('arm64-v8a', 183, 2, 8), 'arm': ('armeabi-v7a', 40, 1, 4), + 'x86_64': ('x86_64', 62, 2, 8)} + +class ProbeError(RuntimeError): + pass + +def stamp() -> str: + return datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ') + +def write_json(path: Path, value: object) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + tmp = path.with_name(path.name + '.tmp') + tmp.write_text(json.dumps(value, indent=2, ensure_ascii=False) + '\n', encoding='utf-8') + os.replace(tmp, path) + +def sha256(path: Path) -> str: + h = hashlib.sha256() + with path.open('rb') as f: + for b in iter(lambda: f.read(1024 * 1024), b''): h.update(b) + return h.hexdigest() + +def git_blob(data: bytes) -> str: + return hashlib.sha1(b'blob ' + str(len(data)).encode() + b'\0' + data).hexdigest() + +def safe_relative(name: str) -> str: + p = PurePosixPath(name) + if not name or p.is_absolute() or '\\' in name or ':' in name or '\x00' in name: + raise ProbeError('Unsafe relative path: ' + repr(name)) + if any(c in ('', '.', '..') for c in name.split('/')) or any(ord(c) < 32 for c in name): + raise ProbeError('Unsafe relative path: ' + repr(name)) + return str(p) + +def run_command(args: list[str], cwd: Path, log: Path, timeout: int = 1800) -> dict: + """No shell, no swallowed exit codes, kill the whole process group on timeout.""" + started = time.monotonic() + log.parent.mkdir(parents=True, exist_ok=True) + with log.open('w', encoding='utf-8') as out: + out.write('$ ' + shlex.join(args) + '\n'); out.flush() + try: + p = subprocess.Popen(args, cwd=cwd, stdout=out, stderr=subprocess.STDOUT, + start_new_session=True) + try: + code = p.wait(timeout=timeout) + status = 'PASS' if code == 0 else 'FAIL' + except (subprocess.TimeoutExpired, KeyboardInterrupt) as e: + os.killpg(p.pid, signal.SIGKILL); p.wait() + if isinstance(e, KeyboardInterrupt): raise + out.write('\nTIMEOUT: entire process group terminated\n') + status, code = 'BLOCKED', 124 + except FileNotFoundError as e: + out.write(str(e)); status, code = 'BLOCKED', 127 + return {'status': status, 'exit_code': code, 'seconds': round(time.monotonic()-started, 3), + 'command': args, 'log': str(log.relative_to(ROOT)) if log.is_relative_to(ROOT) else str(log)} + +def first_errors(log: str, limit: int = 12) -> list[str]: + rows = [s.strip() for s in log.splitlines() if re.search( + r'\b(fatal|error|undefined reference|cannot find|can.t find|not found)\b', s, re.I)] + return rows[:limit] + +def classify(log: str) -> str: + """Triage hint, never an automatic decision to rewrite a subsystem.""" + if re.search(r'cannot find.*(ppcross|ppca64|ppcarm)|compiler.*not found|SDK.*not found', log, re.I): + return 'TOOLCHAIN' + if re.search(r"can.t find unit (Windows|Forms|Controls|Graphics|LCL\w*|Dialogs|ShellAPI)", log, re.I): + return 'PLATFORM_BOUNDARY' + if re.search(r'undefined reference|cannot find -l|error while linking', log, re.I): + return 'NATIVE_LINK_DEPENDENCY' + if re.search(r"can.t find unit|cannot open include file", log, re.I): + return 'SOURCE_OR_PACKAGE_DEPENDENCY' + return 'SOURCE_OR_TOOLCHAIN_REVIEW' diff --git a/tools/decision_report.py b/tools/decision_report.py new file mode 100755 index 0000000..2403ca6 --- /dev/null +++ b/tools/decision_report.py @@ -0,0 +1,84 @@ +"""Collect Task 1 decision-probe evidence. Does not choose architecture.""" +from __future__ import annotations +import json +from pathlib import Path +from common import ROOT, write_json + +QUESTION_MAP={ + 'T1-RP01-Q01':['oq_host_view','oq_host_callback'], + 'T1-RP01-Q02':['decision.schedule.callback_progress','decision.schedule.worker_progress','decision.schedule.lifecycle'], + 'T1-RP01-Q03':['decision.persistence.rename_replace','decision.persistence.append_reopen'], + 'T1-RP02-Q02':['decision.text.cwstring_unit','decision.text.cp1251_decode','decision.text.cp1251_encode'], + 'T1-Q01':['decision.fixture.castle_stream','decision.fixture.materialized_native_path','fixture_staging_driver'], +} + +def load(path:Path): + return json.loads(path.read_text(encoding='utf-8')) + +def main()->int: + latest_path=ROOT/'reports/LATEST.json' + if not latest_path.is_file(): + raise SystemExit('reports/LATEST.json not found; run build first') + latest=load(latest_path);run_id=latest['run_id'];run_dir=ROOT/latest['directory'];build=load(run_dir/'result.json') + devices=[] + for result_path in sorted((ROOT/'reports/device').glob('*/device-result.json')): + try:rec=load(result_path) + except Exception:continue + if rec.get('build_id')!=run_id and rec.get('package',{}).get('build_id')!=run_id:continue + runtime_path=result_path.with_name('runtime.json') + runtime=load(runtime_path) if runtime_path.is_file() else None + devices.append({'directory':str(result_path.parent.relative_to(ROOT)),'result':rec,'runtime':runtime}) + out={'schema':'kam-task1-decision-evidence/2','build_id':run_id,'gate':build.get('gate'), + 'environment':{'toolchain':build.get('toolchain')},'questions':QUESTION_MAP,'profiles':{},'device_runs':[]} + out['infrastructure_files_changed']=['tools/runner.py','tools/artifacts.py','tools/device.py','tools/decision_report.py', + 'config/profiles.json','code/GameViewMain.pas','code/ProbeReport.pas','CODEX_RUN_TASK1_DECISION_TESTS.md','tests/test_probe.py','tests/test_pipeline.py'] + for row in build.get('profiles',[]): + if row.get('id') in ('platform','oq_host_view','oq_host_callback'): + out['profiles'][row['id']]={'compile_link':row.get('compile_link'),'package':row.get('package'),'apk':row.get('apk'),'errors':row.get('errors',[])} + for d in devices: + rec=d['result'];runtime=d['runtime'];entry={'directory':d['directory'],'profile':rec.get('profile',rec.get('package',{}).get('profile')), + 'status':rec.get('status'),'errors':rec.get('errors',[]),'preflight':rec.get('preflight'),'infrastructure_blocker':rec.get('infrastructure_blocker'),'fixture_staging_driver':rec.get('fixture_staging_driver')} + if runtime: + entry['observations']=[x for x in runtime.get('tests',[]) if str(x.get('id','')).startswith('decision.')] + entry['session']=runtime.get('session') + out['device_runs'].append(entry) + out['history']=[d for d in out['device_runs'] if d.get('status')!='AUTOMATIC_SMOKE_PASS'] + valid_runs=[d for d in out['device_runs'] if d.get('status')=='AUTOMATIC_SMOKE_PASS'] + observations={} + for d in valid_runs: + for item in d.get('observations',[]):observations[item.get('id')]=item + if d.get('fixture_staging_driver') is not None:observations['fixture_staging_driver']=d['fixture_staging_driver'] + blocker=next((d.get('infrastructure_blocker',{}).get('reason') for d in reversed(out['device_runs']) if d.get('infrastructure_blocker')),None) + out['decision_test_results']=[{'question_id':q,'test':test,'status':observations.get(test,{}).get('status','NOT_RUN'), + 'evidence':observations.get(test),'blocker':None if test in observations else blocker} for q,tests in QUESTION_MAP.items() for test in tests] + json_path=ROOT/'reports/TASK1-DECISION-EVIDENCE.json';write_json(json_path,out) + lines=['# Task 1 — decision-probe evidence','',f"Build ID: `{run_id}`",'', + 'Этот отчёт фиксирует наблюдения и не закрывает архитектурные вопросы Task 1.','', + '## Environment','',f"- Toolchain receipt: `reports/runs/{run_id}/toolchain.json`",'- Docker command receipts are retained under `reports/runs/*/docker-command.json`.','', + '## Infrastructure files changed',''] + lines += [f'- `{path}`' for path in out['infrastructure_files_changed']] + lines += ['','## Build/package','', '| Profile | Compile/link | APK | Native library inside APK |','|---|---|---|---|'] + for p in ('platform','oq_host_view','oq_host_callback'): + r=out['profiles'].get(p,{});apk=r.get('apk') or {};lines.append(f"| {p} | {r.get('compile_link',{}).get('status','NOT_RUN')} | {r.get('package',{}).get('status','NOT_RUN')} | `{apk.get('main_library_path','')}` |") + lines+=['','## Device observations',''] + if not valid_runs:lines.append('Valid device runs for this build: NOT_RUN.') + for d in valid_runs: + lines.append(f"### {d.get('profile')} — {d.get('status')}") + lines.append('') + if d.get('fixture_staging_driver') is not None:lines.append('fixture_staging_driver: `'+json.dumps(d['fixture_staging_driver'],ensure_ascii=False)+'`') + if d.get('preflight') is not None:lines.append('preflight: `'+json.dumps(d['preflight'],ensure_ascii=False)+'`') + if d.get('infrastructure_blocker') is not None:lines.append('infrastructure_blocker: `'+json.dumps(d['infrastructure_blocker'],ensure_ascii=False)+'`') + for obs in d.get('observations',[]):lines.append(f"- `{obs.get('id')}` — `{obs.get('status')}` — {obs.get('detail','')}") + if d.get('errors'): + for e in d['errors']:lines.append('- error: '+str(e)) + lines.append('') + if out['history']: + lines+=['## Historical blocked/failed attempts',''] + for d in out['history']: + lines.append(f"- `{d.get('directory')}` / {d.get('profile')} — {d.get('status')}: {d.get('infrastructure_blocker') or d.get('errors')}") + lines+=['## Decision-test results','', '| Question / test | Status | Blocker |','|---|---|---|'] + for item in out['decision_test_results']:lines.append(f"| {item['question_id']} / {item['test']} | {item['status']} | {item['blocker'] or ''} |") + md_path=ROOT/'reports/TASK1-DECISION-EVIDENCE.md';md_path.write_text('\n'.join(lines)+'\n',encoding='utf-8') + print(md_path);print(json_path);return 0 + +if __name__=='__main__':raise SystemExit(main()) diff --git a/tools/device.py b/tools/device.py new file mode 100755 index 0000000..8312386 --- /dev/null +++ b/tools/device.py @@ -0,0 +1,134 @@ +"""Install and collect a selected Android decision probe with diagnostic evidence.""" +from __future__ import annotations +import argparse, json, re, shutil, subprocess, time +from pathlib import Path +from common import ROOT, CPUS, ProbeError, safe_relative, sha256, stamp, write_json + +DECISION_PROFILES={'oq_host_view','oq_host_callback'} + +class Adb: + def __init__(self,serial:str|None): self.base=['adb']+(['-s',serial] if serial else []) + def run(self,*args:str,check:bool=True,timeout:int=120)->subprocess.CompletedProcess: + p=subprocess.run(self.base+list(args),capture_output=True,timeout=timeout) + if check and p.returncode: raise ProbeError('ADB command failed: '+' '.join(args)+'\n'+p.stderr.decode(errors='replace')) + return p + def text(self,*args:str,check:bool=True)->str: return self.run(*args,check=check).stdout.decode(errors='replace').strip() + +def own_report_paths(text:str)->list[str]: + paths=[] + for line in text.splitlines(): + name=line.strip()[2:] if line.strip().startswith('./') else line.strip() + try: safe_relative(name) + except ProbeError: continue + if name.endswith('kam-architecture-probe/runtime.json'): paths.append(name) + return paths + +def validate_runtime(data:dict,package:dict,require_frames:bool=True)->list[str]: + errors=[] + for key,value in (('schema','kam-android-runtime/1'),('build_id',package['build_id']),('profile',package['profile']),('cpu_expected',package['cpu'])): + if data.get(key)!=value: errors.append('Wrong/stale runtime '+key) + if data.get('automatic_smoke_status')!='PASS': errors.append('Automatic smoke did not pass') + tests=data.get('tests') + if not isinstance(tests,list) or not tests: return errors+['No runtime tests'] + by_id={x.get('id'):x for x in tests if isinstance(x,dict)}; profile=package['profile']; required=['android.startup']; observed=[] + if profile not in DECISION_PROFILES: + required += ['abi.pointer','abi.scalar_layout','abi.endianness','files.native_utf8','files.seek','files.case_sensitive','fpc.zstream','fpc.thread'] + observed += ['decision.persistence.rename_replace','decision.persistence.append_reopen'] + if profile=='oq_host_view': required += ['decision.host.view'] + elif profile=='oq_host_callback': required += ['decision.host.callback'] + if profile=='platform': required += ['decision.text.cwstring_unit']; observed += ['decision.text.cp1251_decode','decision.text.cp1251_encode'] + if require_frames: required += ['render.frame_callbacks','render.context'] + for name in required: + if by_id.get(name,{}).get('status')!='PASS': errors.append('Required runtime check did not pass: '+name) + for name in observed: + if name not in by_id: errors.append('Decision observation missing: '+name) + if any(t.get('status')=='FAIL' for t in tests if isinstance(t,dict)): errors.append('One or more runtime checks failed') + if require_frames and data.get('session',{}).get('frames',0)<30: errors.append('Fewer than 30 frames') + return errors + +def read_runtime(adb:Adb,pkg:str)->tuple[dict|None,str|None]: + paths=own_report_paths(adb.text('shell','run-as',pkg,'find','.','-name','runtime.json',check=False)) + if len(paths)!=1: return None,None + raw=adb.run('exec-out','run-as',pkg,'cat',paths[0],check=False).stdout + try: return json.loads(raw),paths[0] + except (ValueError,UnicodeDecodeError): return None,None + +def wait_runtime(adb:Adb,pkg:str,predicate,timeout:int)->tuple[dict|None,str|None]: + end=time.monotonic()+timeout + while time.monotonic()None: + out.joinpath(name).write_text('exit='+str(p.returncode)+'\nstdout:\n'+p.stdout.decode(errors='replace')+'\nstderr:\n'+p.stderr.decode(errors='replace'),encoding='utf-8') + +def main()->int: + ap=argparse.ArgumentParser(description=__doc__) + ap.add_argument('--profile',default='platform');ap.add_argument('--serial');ap.add_argument('--index',type=Path,default=ROOT/'dist/LATEST.json') + ap.add_argument('--seconds',type=int,default=30);ap.add_argument('--exercise-lifecycle',action='store_true');ap.add_argument('--stage-file',type=Path) + a=ap.parse_args() + if a.seconds<5: ap.error('--seconds must be at least 5') + out=ROOT/'reports/device'/stamp(); n=1 + while out.exists(): out=ROOT/'reports/device'/(stamp()+'-'+str(n));n+=1 + out.mkdir(parents=True) + record={'schema':'kam-android-device/2','status':'BLOCKED','errors':[],'preflight':{'status':'NOT_RUN'},'manual_visual_touch_audio':'NOT_VERIFIED','world_simulation':'NOT_RUN','command_environment':{'host_adb':shutil.which('adb'),'timeout_seconds':a.seconds}} + adb=None + try: + index=json.loads(a.index.read_text()); matches=[p for p in index.get('packages',[]) if p.get('profile')==a.profile] + if len(matches)!=1: raise ProbeError('No unique successfully-built APK for '+a.profile) + package=matches[0]; apk=ROOT/package['file']; pkg=package['package'];record.update(build_id=package['build_id'],profile=a.profile,package=package) + if not apk.is_file() or sha256(apk)!=package['sha256']: raise ProbeError('APK missing/changed') + if not re.fullmatch(r'org\.kammobile\.archprobe\.[a-z0-9]+',pkg): raise ProbeError('Refusing unrelated Android package') + if not shutil.which('adb'): record['preflight']={'status':'BLOCKED','reason':'adb is not installed on host'};raise ProbeError(record['preflight']['reason']) + listed=Adb(None).text('devices').splitlines()[1:]; devices=[line.split()[:2] for line in listed if line.strip()] + record['preflight']={'adb':'PASS','devices':devices} + if len(devices)!=1 or len(devices[0])!=2 or devices[0][1]!='device': record['preflight'].update(status='BLOCKED',reason='Require exactly one authorized device; found '+repr(devices));raise ProbeError(record['preflight']['reason']) + if a.serial and a.serial!=devices[0][0]: record['preflight'].update(status='BLOCKED',reason='Requested serial is not the sole authorized device');raise ProbeError(record['preflight']['reason']) + adb=Adb(devices[0][0]); record['device']={'serial':devices[0][0],'abi':adb.text('shell','getprop','ro.product.cpu.abilist'),'android':adb.text('shell','getprop','ro.build.version.release'),'sdk':adb.text('shell','getprop','ro.build.version.sdk'),'model':adb.text('shell','getprop','ro.product.model')} + if CPUS[package['cpu']][0] not in record['device']['abi'].split(','): record['preflight'].update(status='BLOCKED',reason='Device ABI does not support arm64-v8a');raise ProbeError(record['preflight']['reason']) + record['preflight']['status']='PASS' + install=adb.run('install','-r',str(apk),check=False);save_command(out,'install.log',install);record['install']={'package_id':pkg,'initial_exit':install.returncode,'result':install.stdout.decode(errors='replace').strip()} + if install.returncode and b'INSTALL_FAILED_UPDATE_INCOMPATIBLE' in install.stdout+install.stderr: + uninstall=adb.run('uninstall',pkg,check=False);save_command(out,'uninstall.log',uninstall);record['install']['uninstall_for_ephemeral_key']={'exit':uninstall.returncode,'result':uninstall.stdout.decode(errors='replace').strip()} + install=adb.run('install',str(apk),check=False);save_command(out,'install-retry.log',install);record['install'].update(retry_exit=install.returncode,retry_result=install.stdout.decode(errors='replace').strip()) + if install.returncode: raise ProbeError('APK install failed; see install.log') + run_as=adb.run('shell','run-as',pkg,'id',check=False);save_command(out,'run-as.log',run_as);record['run_as']={'status':'PASS' if run_as.returncode==0 else 'BLOCKED','output':run_as.stdout.decode(errors='replace').strip(),'error':run_as.stderr.decode(errors='replace').strip()} + if run_as.returncode: record['infrastructure_blocker']={'stage':'run_as_preflight','reason':'run-as unavailable for '+pkg};raise ProbeError(record['infrastructure_blocker']['reason']) + if a.stage_file: + if not a.stage_file.is_file(): raise ProbeError('Stage file not found: '+str(a.stage_file)) + rel='files/kam-architecture-probe/driver-staged-fixture.bin';adb.run('shell','run-as',pkg,'sh','-c','mkdir -p files/kam-architecture-probe') + p=subprocess.run(adb.base+['exec-in','run-as',pkg,'sh','-c','cat > '+rel],input=a.stage_file.read_bytes(),stdout=subprocess.PIPE,stderr=subprocess.PIPE);save_command(out,'stage.log',p) + size=adb.text('shell','run-as',pkg,'stat','-c','%s',rel,check=False);record['fixture_staging_driver']={'status':'PASS' if p.returncode==0 and size==str(a.stage_file.stat().st_size) else 'FAIL','source':str(a.stage_file),'bytes':a.stage_file.stat().st_size,'device_bytes':size} + adb.run('shell','am','force-stop',pkg); listing=adb.text('shell','run-as',pkg,'find','.','-name','runtime.json',check=False) + for name in own_report_paths(listing): adb.run('shell','run-as',pkg,'rm','--',name) + adb.run('logcat','-c');record['launch_marker_epoch_ms']=int(time.time()*1000) + component=adb.text('shell','cmd','package','resolve-activity','--brief','-a','android.intent.action.MAIN','-c','android.intent.category.LAUNCHER',pkg); candidates=[x.strip() for x in component.splitlines() if x.strip().startswith(pkg+'/')] + if len(candidates)!=1 or not re.fullmatch(r'[\w.$/]+',candidates[0]): raise ProbeError('Cannot resolve probe launch Activity') + launch=adb.run('shell','am','start','-W','-n',candidates[0],check=False);save_command(out,'launch.log',launch);record['launch']={'activity':candidates[0],'exit':launch.returncode,'result':launch.stdout.decode(errors='replace').strip()} + if launch.returncode: raise ProbeError('Launch failed; see launch.log') + data,path=wait_runtime(adb,pkg,lambda d:any(x.get('id')=='android.startup' and x.get('status')=='PASS' for x in d.get('tests',[])),a.seconds) + if data is None: raise ProbeError('android.startup was not observed before timeout; see logcat.log and process.log') + if a.exercise_lifecycle: + adb.run('shell','input','keyevent','KEYCODE_HOME');time.sleep(1);adb.run('shell','am','start','-W','-n',candidates[0]) + expected=[] + if a.exercise_lifecycle and a.profile=='platform': expected=['decision.schedule.callback_progress','decision.schedule.worker_progress','decision.schedule.lifecycle'] + if a.profile=='platform': expected += ['render.frame_callbacks','decision.text.cwstring_unit','decision.text.cp1251_decode','decision.text.cp1251_encode'] + if expected: data,path=wait_runtime(adb,pkg,lambda d: all(any(x.get('id')==name for x in d.get('tests',[])) for name in expected),a.seconds) + if data is None: raise ProbeError('Runtime report disappeared; see logcat.log and process.log') + out.joinpath('runtime.json').write_text(json.dumps(data,ensure_ascii=False,indent=2)+'\n',encoding='utf-8');record['runtime_path']=path + record['errors']+=validate_runtime(data,package,require_frames=package['profile'] not in DECISION_PROFILES) + if expected and not all(any(x.get('id')==name for x in data.get('tests',[])) for name in expected): record['errors'].append('Timed out waiting for observations: '+', '.join(expected)) + if a.stage_file and record.get('fixture_staging_driver',{}).get('status')!='PASS': record['errors'].append('Existing server fixture staging size mismatch') + record['status']='FAIL' if record['errors'] else 'AUTOMATIC_SMOKE_PASS' + except (ProbeError,OSError,ValueError,KeyError,subprocess.TimeoutExpired) as e: + record['errors'].append(str(e)) + if record['preflight'].get('status')=='BLOCKED': record['infrastructure_blocker']={'stage':'device_preflight','reason':record['preflight'].get('reason',str(e))} + finally: + if adb: + p=adb.run('shell','pidof',record.get('package',{}).get('package',''),check=False);save_command(out,'process.log',p);record['process_after_launch']=p.stdout.decode(errors='replace').strip() + p=adb.run('logcat','-d','-v','threadtime',check=False);save_command(out,'logcat.log',p) + write_json(out/'device-result.json',record);print(record['status']+'\n'+str(out));[print(x) for x in record['errors']] + return 0 if record['status']=='AUTOMATIC_SMOKE_PASS' else 1 if record['status']=='FAIL' else 2 +if __name__=='__main__': raise SystemExit(main()) diff --git a/tools/generate.py b/tools/generate.py new file mode 100755 index 0000000..c71c973 --- /dev/null +++ b/tools/generate.py @@ -0,0 +1,129 @@ +"""Generate isolated Android CGE targets. No KaM stubs or rewritten game loaders.""" +from __future__ import annotations +import json, os, shutil, urllib.parse, xml.etree.ElementTree as ET +from pathlib import Path +from common import ROOT, COMMIT, VERSION, CPUS, ProbeError, safe_relative, sha256, write_json +from source_audit import search_paths + +UPSTREAM_DEFINES=['DBG_SKIP_SECURE_AUTH'] # Existing upstream switch; private multiplayer authentication is out of scope. +COMPILER_OPTIONS=['-B','-CX-','-XX-','-k--no-undefined','-k--no-gc-sections'] + +def imports_source(profile:dict)->tuple[str,list[str]]: + units=list(profile['units']) + points=profile['id'] in ('core_types','core_streams','integration') + streams=profile['id'] in ('core_streams','integration') + if points and 'KM_Points' not in units:units.append('KM_Points') + head='unit ProbeImports;\n{$mode delphi}{$H+}\ninterface\nuses ProbeReport;\nprocedure RunUpstreamChecks(R:TProbeReport);\nimplementation\n' + head+='uses SysUtils'+(' ,'+', '.join(units) if units else '')+';\n' + head+='procedure RunUpstreamChecks(R:TProbeReport);\n' + if points: + head+='var P,Q:TKMPoint;\n' + if streams:head+=' S:TKMemoryStreamBinary; I:Integer; U:UnicodeString;\n' + head+='begin\n' + if not units: + head+=" R.Add('upstream.imports','NOT_RUN','Platform-only APK; no KaM modules imported');\n" + else: + head+=" R.Add('upstream.imports','PASS','Real imported units initialized: "+', '.join(units)+"; this does not execute a game world');\n" + if points: + head+=''' try + P:=KMPoint(17,-3); Q:=KMPointRight(P); + R.Check('core.points',(P.X=17) and (P.Y=-3) and (Q.X=18) and (Q.Y=-3), + 'Calls original KMPoint / KMPointRight, not a local replacement'); + R.Check('core.point_layout',(SizeOf(TKMPoint)=8) and (SizeOf(TKMPointF)=8), + Format('TKMPoint=%d TKMPointF=%d',[SizeOf(TKMPoint),SizeOf(TKMPointF)])); + except on E:Exception do R.Add('core.points','FAIL',E.ClassName+': '+E.Message);end; +''' + if streams: + head+=''' try + S:=TKMemoryStreamBinary.Create; + try + S.Write(Integer(123456)); S.Write(P); S.WriteW(UnicodeString('KaM Android')); + S.Position:=0; S.Read(I); S.Read(Q); S.ReadW(U); + R.Check('core.stream_roundtrip',(I=123456) and (Q.X=P.X) and (Q.Y=P.Y) + and (U=UnicodeString('KaM Android')) and (S.Position=S.Size), + 'Calls original TKMemoryStreamBinary Read/Write/ReadW/WriteW'); + finally S.Free;end; + except on E:Exception do R.Add('core.stream_roundtrip','FAIL',E.ClassName+': '+E.Message);end; +''' + head+='end;\nend.\n' + return head,units + +def asset_inventory(root:Path|None, includes:list[str]|None=None)->list[dict]: + if root is None:return [] + root=root.resolve() + if not root.is_dir():raise ProbeError('Asset directory does not exist: '+str(root)) + if includes: + result=[] + seen=set() + for raw in includes: + rel=safe_relative(raw.replace('\\','/')) + if rel in seen:continue + seen.add(rel);p=root/rel + if p.is_symlink():raise ProbeError('Asset symlinks are not allowed: '+str(p)) + if not p.is_file():raise ProbeError('Requested asset path not found under asset root: '+rel) + result.append({'path':rel,'size':p.stat().st_size,'sha256':sha256(p), + 'url':'castle-data:/game/'+urllib.parse.quote(rel,safe='/')}) + return result + result=[] + for p in sorted(root.rglob('*')): + if p.is_symlink():raise ProbeError('Asset symlinks are not allowed: '+str(p)) + if p.is_dir():continue + if not p.is_file():raise ProbeError('Not a regular asset: '+str(p)) + rel=safe_relative(p.relative_to(root).as_posix()) + if rel=='README.md' or '.git' in p.parts:continue + result.append({'path':rel,'size':p.stat().st_size,'sha256':sha256(p), + 'url':'castle-data:/game/'+urllib.parse.quote(rel,safe='/')}) + return result + +def make_target(project:Path,profile:dict,cpu:str,build_id:str,source:Path|None, + extras:dict,assets_root:Path|None=None,assets:list[dict]|None=None)->dict: + if project.exists():raise ProbeError('Target directory must be fresh: '+str(project)) + project.mkdir(parents=True) + shutil.copytree(ROOT/'code',project/'code') + text,units=imports_source(profile) + (project/'code/ProbeImports.pas').write_text(text,encoding='utf-8') + (project/'code/ProbeBuild.pas').write_text( + 'unit ProbeBuild;\n{$mode objfpc}{$H+}\ninterface\nconst\n'+ + f" ProbeBuildId='{build_id}';\n ProbeProfile='{profile['id']}';\n ProbeCpu='{cpu}';\n"+ + f" ProbeUpstreamCommit='{COMMIT}';\n ProbePointerBytes={CPUS[cpu][3]};\nimplementation\nend.\n") + data=project/'data';data.mkdir() + entries=assets or [] + if entries and assets_root is None:raise ProbeError('Missing asset root') + for e in entries: + dest=data/'game'/e['path'];dest.parent.mkdir(parents=True,exist_ok=True) + shutil.copyfile(assets_root/e['path'],dest) + if sha256(dest)!=e['sha256']:raise ProbeError('Asset changed while staging: '+e['path']) + write_json(data/'assets-manifest.json',{'schema':'kam-probe-assets/1','files':entries}) + name='kam_probe_'+profile['id'] + root=ET.Element('project',name=name,caption='KaM Android: '+profile['id'], + author='KaM Mobile',qualified_name='org.kammobile.archprobe.'+profile['id'].replace('_',''), + game_units=profile.get('game_unit','GameInitialize'),compiler='fpc',screen_orientation='landscape',fullscreen_immersive='true') + ET.SubElement(root,'version',value=VERSION,code='30') + options=ET.SubElement(root,'compiler_options');paths=ET.SubElement(options,'search_paths') + ET.SubElement(paths,'path',value='code/') + include=[] + if units: + if source is None:raise ProbeError('KaM source is required for '+profile['id']) + include=search_paths(source) + include.extend(Path(p).resolve() for p in extras.get('source_paths',[])) + for p in include: + if not p.is_dir():raise ProbeError('Source path missing: '+str(p)) + ET.SubElement(paths,'path',value=os.path.relpath(p,project).replace(os.sep,'/')) + defines=ET.SubElement(options,'defines') + for define in (UPSTREAM_DEFINES if units else []):ET.SubElement(defines,'define').text=define + custom=ET.SubElement(options,'custom_options') + for flag in COMPILER_OPTIONS:ET.SubElement(custom,'option').text=flag + libraries=ET.SubElement(options,'library_paths') + for p in extras.get('library_paths',[]): + p=Path(p).resolve() + if not p.is_dir():raise ProbeError('Native library path missing: '+str(p)) + ET.SubElement(libraries,'path',value=os.path.relpath(p,project).replace(os.sep,'/')) + # Android SDK versions deliberately inherited from the locked CGE image. + ET.indent(root) + ET.ElementTree(root).write(project/'CastleEngineManifest.xml',encoding='utf-8',xml_declaration=True) + description={'profile':profile['id'],'name':name,'package':'org.kammobile.archprobe.'+profile['id'].replace('_',''), + 'cpu':cpu,'units':units,'game_unit':profile.get('game_unit','GameInitialize'),'source_paths':[str(p) for p in include], + 'compiler_options':COMPILER_OPTIONS,'upstream_defines':UPSTREAM_DEFINES if units else [], + 'assets':entries,'shims':False} + write_json(project/'target.json',description) + return description diff --git a/tools/runner.py b/tools/runner.py new file mode 100755 index 0000000..a387fbd --- /dev/null +++ b/tools/runner.py @@ -0,0 +1,307 @@ +"""One-command Android-only compile, full native link, APK and evidence pipeline.""" +from __future__ import annotations +import argparse, json, os, re, shutil, subprocess, sys, uuid, zipfile +from pathlib import Path +from common import ROOT, COMMIT, CPUS, ProbeError, stamp, sha256, write_json, run_command, first_errors, classify +from source_fetch import fetch, verify +from source_audit import inventory +from generate import make_target, asset_inventory +from artifacts import inspect_elf, inspect_apk + +IMAGE_TAG='kambi/castle-engine-cloud-builds-tools:cge-unstable' +PROFILES=json.loads((ROOT/'config/profiles.json').read_text()) + +def selected_profiles(text:str)->list[dict]: + if text=='all':return PROFILES + names=[s.strip() for s in text.split(',') if s.strip()] + available={p['id'] for p in PROFILES} + if not names or set(names)-available:raise ProbeError('Unknown/empty profiles: '+text) + names=set(names)|{'platform'} # Always establish the same-toolchain platform baseline. + return [p for p in PROFILES if p['id'] in names] + +def cge_command(cpu:str,action:str)->list[str]: + if cpu not in CPUS:raise ProbeError('Unsupported Android CPU') + if action not in ('compile','package'):raise ProbeError('Not an Android build action') + cmd=['castle-engine','--os=android','--cpu='+cpu,'--compiler=fpc','--mode=debug'] + if action=='package':cmd+=['--assume-compiled','--package-format=android-apk'] + return cmd+[action] + +def blocked_row(profile:dict,reason:str)->dict: + return {'id':profile['id'],'required':profile['required'],'purpose':profile['purpose'], + 'compile_link':{'status':'BLOCKED','reason':reason},'package':{'status':'NOT_RUN'}, + 'runtime':{'status':'NOT_RUN'},'errors':[]} + +def gate(rows:list[dict])->str: + by_id={r['id']:r for r in rows} + required=[p for p in PROFILES if p['required']] + if any(r['compile_link']['status']=='FAIL' or r['package']['status']=='FAIL' for r in rows if r['required']): + return 'NOT_BUILDABLE_AS_IS' + if any(r['compile_link']['status']=='BLOCKED' for r in rows if r['required']):return 'BLOCKED' + if any(p['id'] not in by_id for p in required):return 'PARTIAL_SELECTION' + for p in required: + r=by_id[p['id']] + if r['compile_link']['status']!='PASS':return 'BLOCKED' + if p['package'] and r['package']['status']!='PASS':return 'BLOCKED' + return 'ANDROID_BUILD_GATE_PASS' + +def render_summary(result:dict,out:Path)->None: + title='# KaM r16020 — Android architecture probe\n\n' + title+=f"Build ID: `{result['run_id']}` · CPU: `{result['cpu']}` · Gate: **{result['gate']}**\n\n" + title+='Нет desktop target. Успех означает сборку выбранных Android-корней, не игровую совместимость.\n\n' + title+='| Проба | Compile + native link | APK | Ошибка / следующий шаг |\n|---|---|---|---|\n' + for r in result['profiles']: + error=(r.get('errors') or [r['compile_link'].get('reason','')])[0] + error=error.replace('|','/').replace('\n',' ')[:260] + title+=f"| {r['id']} | {r['compile_link']['status']} | {r['package']['status']} | {error} |\n" + title+='\n## Что установлено\n\n' + title+='`PASS` compile_link = FPC завершился успешно, найден свежий ELF нужного ABI, проверены корневые PPU. Линковка не отключена; требуются определённые символы. ' + title+='`PASS` APK = упаковка завершилась, проверены ZIP/DEX, правильный ABI, совпадение native library и DT_NEEDED. ' + title+='Подпись проверяется отдельно, если доступен apksigner. `runtime=NOT_RUN` до реального запуска на Android.\n\n' + title+='`NOT_BUILDABLE_AS_IS` — найдено препятствие в текущем коде/окружении, а не доказательство, что core надо переписать. ' + title+='Ошибки legacy_render/legacy_audio диагностические и не входят в обязательный gate, но их транзитивное попадание в core всё равно видно.\n\n' + title+='## Как принимать решение\n\n' + title+='| Свидетельство | Разумное решение |\n|---|---|\n' + title+='| Корень собирается с оригинальными исходниками | Кандидат на reuse; поведение ещё проверяется отдельно |\n' + title+='| Активная зависимость Windows/LCL/desktop GL | Исследовать минимальное отделение платформенной части |\n' + title+='| Отсутствует source unit / package | Проверить комплектность исходников, условные defines и пакет для Android |\n' + title+='| Undefined reference / отсутствует native library | Сборка/подключение Android-библиотеки либо адаптер Castle |\n' + title+='| Один и тот же блокер у многих модулей | Исправить общий корень и повторить пробы; не считать это множеством отдельных переписываний |\n' + title+='| Срабатывает exception при запуске | Снять logcat; проверить unit initialization/lifecycle/доступ к данным |\n' + title+='\nНи один автоматический результат здесь не выносит решение «написать подсистему с нуля».\n' + (out/'SUMMARY.md').write_text(title,encoding='utf-8') + +def finalize(result:dict,out:Path)->int: + result['gate']=gate(result['profiles']) + write_json(out/'result.json',result);render_summary(result,out) + write_json(ROOT/'reports/LATEST.json',{'run_id':result['run_id'],'directory':str(out.relative_to(ROOT)), + 'gate':result['gate'],'summary':str((out/'SUMMARY.md').relative_to(ROOT))}) + shutil.copyfile(out/'SUMMARY.md',ROOT/'reports/LATEST-SUMMARY.md') + # Evidence intentionally excludes APK/.so, user game assets, caches and source trees. + with zipfile.ZipFile(out/'evidence.zip','w',zipfile.ZIP_DEFLATED) as z: + for p in sorted(out.rglob('*')): + if p.is_file() and p.suffix in ('.json','.log','.md','.txt','.xml','.pas'): + z.write(p,p.relative_to(out)) + print('\n'+result['gate']) + print('Report: '+str(out/'SUMMARY.md')) + print('Evidence (no game assets): '+str(out/'evidence.zip')) + return 0 if result['gate']=='ANDROID_BUILD_GATE_PASS' else 1 if result['gate']=='NOT_BUILDABLE_AS_IS' else 2 + +def new_result(cpu:str,profiles:list[dict],reason:str='Not started')->tuple[dict,Path]: + rid=stamp()+'-'+uuid.uuid4().hex[:8] + out=ROOT/'reports/runs'/rid;out.mkdir(parents=True) + return ({'schema':'kam-android-build/1','run_id':rid,'cpu':cpu,'commit':COMMIT, + 'desktop_target':False,'shims':False,'runtime':'NOT_RUN', + 'profiles':[blocked_row(p,reason) for p in profiles]},out) + +def extra_paths()->dict: + d=json.loads((ROOT/'config/extra-paths.json').read_text()) + for key in ('source_paths','library_paths'): + if not isinstance(d.get(key),list) or not all(isinstance(s,str) for s in d[key]): + raise ProbeError('extra-paths.json must contain string arrays') + d[key]=[str((ROOT/p).resolve()) if not Path(p).is_absolute() else str(Path(p).resolve()) for p in d[key]] + return d + +def tool_versions(out:Path)->dict: + commands={'castle':['castle-engine','--version'],'fpc':['fpc','-iV'],'java':['java','-version'], + 'adb':['adb','version']} + d={} + for name,cmd in commands.items(): + if shutil.which(cmd[0]): + rec=run_command(cmd,ROOT,out/(name+'-version.log'),60) + rec['output']=(out/(name+'-version.log')).read_text(errors='replace')[:16000];d[name]=rec + else:d[name]={'status':'NOT_INSTALLED'} + d['paths']={k:os.environ[k] for k in ('CASTLE_ENGINE_PATH','ANDROID_HOME','ANDROID_SDK_ROOT', + 'ANDROID_NDK_ROOT','JAVA_HOME','FPCDIR') if k in os.environ} + for variable in ('ANDROID_NDK_ROOT','ANDROID_HOME','ANDROID_SDK_ROOT'): + if os.environ.get(variable): + p=Path(os.environ[variable])/'source.properties' + if p.is_file():d[variable+'_source_properties']=p.read_text(errors='replace') + lock=ROOT/'config/toolchain.lock.json' + if lock.is_file():d['docker_image_lock']=json.loads(lock.read_text()) + write_json(out/'toolchain.json',d) + return d + +def signature_tool()->str|None: + direct=shutil.which('apksigner') + if direct:return direct + paths=[] + for key in ('ANDROID_HOME','ANDROID_SDK_ROOT'): + if os.environ.get(key):paths+=list(Path(os.environ[key]).glob('build-tools/*/apksigner')) + return str(sorted(paths)[-1]) if paths else None + +def build_local(a:argparse.Namespace,profiles:list[dict])->int: + result,out=new_result(a.cpu,profiles) + result['toolchain']=tool_versions(out) + if not shutil.which('castle-engine'): + for r in result['profiles']:r['compile_link']['reason']='castle-engine not installed; no Android compilation attempted' + return finalize(result,out) + source=ROOT/'vendor/kam_remake';audit=None;source_error='';assets=[] + need_source=any(p['units'] for p in profiles) + try: + extras=extra_paths() + assets=asset_inventory(Path(a.assets),a.asset_path) if a.assets else [] + result['asset_count']=len(assets) + if need_source: + if (source/'SOURCE_RECEIPT.json').is_file():receipt=verify(source) + elif a.no_fetch:raise ProbeError('Pinned source tree absent and --no-fetch requested') + else:receipt=fetch(source) + write_json(out/'source-receipt.json',receipt) + audit=inventory(source);write_json(out/'source-inventory.json',audit) + for path in extras['source_paths']: + for p in Path(path).rglob('*.pas'): + if p.stem.casefold() in audit['units']: + raise ProbeError('Extra path shadows original unit '+p.stem+'; no shims allowed') + except (ProbeError,OSError,ValueError,KeyError) as e: + source_error=str(e);extras={'source_paths':[],'library_paths':[]} + result['source_or_inputs_error']=source_error + work=ROOT/'build'/result['run_id'];work.mkdir(parents=True) + dist=ROOT/'dist'/result['run_id'];dist.mkdir(parents=True) + package_index=[] + for idx,profile in enumerate(profiles): + row=result['profiles'][idx];ident=profile['id'];project=work/ident + print(f"[{idx+1}/{len(profiles)}] Android {a.cpu}: {ident}",flush=True) + try: + if profile['units']: + if source_error:raise ProbeError(source_error) + missing=[u for u in profile['units'] if u.casefold() not in audit['units']] + if missing:raise ProbeError('Missing required original source units: '+', '.join(missing)) + # Resources only enter APKs when explicitly requested. The platform baseline remains independent. + pack=profile['package'] or a.package_all + use_assets=bool(a.assets) and pack and ident=='platform' + description=make_target(project,profile,a.cpu,result['run_id'],source if profile['units'] else None, + extras if profile['units'] else {'source_paths':[],'library_paths':[]}, + Path(a.assets) if use_assets else None,assets if use_assets else []) + row['target']=description + evidence=out/ident;evidence.mkdir() + for file in ('CastleEngineManifest.xml','target.json'): + shutil.copyfile(project/file,evidence/file) + shutil.copyfile(project/'code/ProbeImports.pas',evidence/'ProbeImports.pas') + row['compile_link']=run_command(cge_command(a.cpu,'compile'),project,evidence/'compile-link.log',a.timeout) + text=(evidence/'compile-link.log').read_text(errors='replace') + if row['compile_link']['status']!='PASS': + row['errors']=first_errors(text) or [text[-1500:]];row['triage']=classify(text);continue + name='lib'+description['name']+'.so' + outputs=list(project.rglob('lib'+description['name']+'*.so')) + if not outputs:raise ProbeError('Compiler exited 0 but no expected native library matching: '+name) + hashes={sha256(p) for p in outputs} + if len(hashes)!=1:raise ProbeError('Ambiguous different native outputs') + library=outputs[0];name=library.name;elf=inspect_elf(library.read_bytes(),a.cpu) + row['native_library']={'file':str(library.relative_to(ROOT)),'sha256':sha256(library),'elf':elf} + ppu_files={p.stem.casefold():p for p in project.rglob('*.ppu')} + evidence_units=[] + for unit in description['units']: + p=ppu_files.get(unit.casefold()) + if p is None or p.read_bytes()[:3]!=b'PPU': + raise ProbeError('No fresh Pascal unit artifact found for '+unit) + evidence_units.append({'unit':unit,'ppu':str(p.relative_to(ROOT)),'sha256':sha256(p), + 'source_candidates':audit['units'].get(unit.casefold(),[])}) + row['compiled_roots']=evidence_units + if not pack:continue + row['package']=run_command(cge_command(a.cpu,'package'),project,evidence/'package.log',a.timeout) + if row['package']['status']!='PASS': + row['errors']+=first_errors((evidence/'package.log').read_text(errors='replace'));continue + candidates=list(project.rglob('*.apk')) + if not candidates:raise ProbeError('Packaging exited 0 but no new APK') + apk_hashes={sha256(p) for p in candidates} + if len(apk_hashes)!=1:raise ProbeError('Multiple different APK outputs; refusing arbitrary selection') + apk=candidates[0] + apk_info=inspect_apk(apk,a.cpu,name,row['native_library']['sha256'],library.read_bytes()) + signer=signature_tool() + if signer: + sign=run_command([signer,'verify','--verbose',str(apk)],project,evidence/'signature.log',120) + row['signature']=sign + if sign['status']!='PASS':row['package']['status']='FAIL';continue + else:row['signature']={'status':'NOT_RUN','reason':'apksigner not found'} + final=dist/(description['name']+'-'+a.cpu+'-debug.apk');shutil.copy2(apk,final) + row['apk']={'file':str(final.relative_to(ROOT)),**apk_info} + package_index.append({'profile':ident,'package':description['package'],'file':str(final.relative_to(ROOT)), + 'sha256':sha256(final),'cpu':a.cpu,'build_id':result['run_id'], + 'native_library_path_in_apk':apk_info['main_library_path']}) + except (ProbeError,OSError,ValueError,KeyError,zipfile.BadZipFile) as e: + row['errors'].append(str(e)) + # A supposedly successful compiler/packager without valid output is a real test failure. + if row['package']['status']=='PASS':row['package']['status']='FAIL' + elif row['compile_link']['status']=='PASS':row['compile_link']['status']='FAIL' + else:row['compile_link']={'status':'BLOCKED','reason':str(e)} + finally: + write_json(out/'result-in-progress.json',result) + if need_source and not source_error: + try:verify(source) + except ProbeError as e: + result['source_modified_during_build']=str(e) + for r in result['profiles']: + if r['id']!='platform':r['compile_link']['status']='FAIL';r['errors'].append(str(e)) + write_json(dist/'apks.json',{'build_id':result['run_id'],'packages':package_index}) + write_json(ROOT/'dist/LATEST.json',{'build_id':result['run_id'],'directory':str(dist.relative_to(ROOT)), + 'packages':package_index}) + return finalize(result,out) + +def docker_build(a:argparse.Namespace,profiles:list[dict])->int: + result,out=new_result(a.cpu,profiles,'Docker bootstrap not completed') + if not shutil.which('docker'): + for r in result['profiles']:r['compile_link']['reason']='Docker not installed. Use --local only with a configured CGE/FPC Android toolchain.' + return finalize(result,out) + try: + lock=ROOT/'config/toolchain.lock.json' + if lock.is_file(): + image=json.loads(lock.read_text())['image'] + if not re.fullmatch(r'[\w./-]+@sha256:[a-f0-9]{64}',image):raise ProbeError('Invalid toolchain digest lock') + p=subprocess.run(['docker','image','inspect',image],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL) + if p.returncode: + r=run_command(['docker','pull','--platform=linux/amd64',image],ROOT,out/'docker-pull.log',a.timeout) + if r['status']!='PASS':raise ProbeError('Cannot obtain locked Docker image; see docker-pull.log') + else: + r=run_command(['docker','pull','--platform=linux/amd64',IMAGE_TAG],ROOT,out/'docker-pull.log',a.timeout) + if r['status']!='PASS':raise ProbeError('Docker image pull failed; see docker-pull.log') + raw=subprocess.check_output(['docker','image','inspect',IMAGE_TAG],text=True) + info=json.loads(raw)[0];digests=info.get('RepoDigests',[]) + image=next((d for d in digests if 'castle-engine-cloud-builds-tools@sha256:' in d),None) + if image is None:raise ProbeError('Image has no immutable RepoDigest') + write_json(lock,{'image':image,'image_id':info['Id'],'resolved_from':IMAGE_TAG,'locked_at':stamp(), + 'note':'Subsequent runs use this digest, never silently advance the tag.'}) + cache=ROOT/'.cache/home';cache.mkdir(parents=True,exist_ok=True) + cmd=['docker','run','--rm','--platform=linux/amd64','--user','0:0', + '-e','HOME=/work/.cache/home','-v',str(ROOT)+':/work','-w','/work'] + if os.environ.get('GITHUB_TOKEN'):cmd+=['-e','GITHUB_TOKEN'] + if a.assets: + resource=Path(a.assets).resolve() + if not resource.is_dir():raise ProbeError('Asset directory not found: '+str(resource)) + cmd+=['-v',str(resource)+':/probe-assets:ro'] + cmd+=['--entrypoint','python3',image,'tools/runner.py','--local','--cpu',a.cpu, + '--profiles',a.profiles,'--timeout',str(a.timeout)] + if a.no_fetch:cmd+=['--no-fetch'] + if a.package_all:cmd+=['--package-all'] + if a.assets:cmd+=['--assets','/probe-assets'] + for rel in a.asset_path or []:cmd+=['--asset-path',rel] + write_json(out/'docker-command.json',{'command':cmd,'image':image}) + # Stream the user's real build directly. No background processes/promises. + latest=ROOT/'reports/LATEST.json' + before=latest.read_bytes() if latest.exists() else None + code=subprocess.call(cmd,cwd=ROOT) + after=latest.read_bytes() if latest.exists() else None + if code in (0,1,2) and after is not None and after!=before: + fresh=json.loads(after) + if (ROOT/fresh['directory']/'result.json').is_file():return code + raise ProbeError('Container could not finish the report; Docker exit code '+str(code)) + except (ProbeError,OSError,ValueError,KeyError,subprocess.CalledProcessError) as e: + for r in result['profiles']:r['compile_link']['reason']=str(e) + return finalize(result,out) + +def main()->int: + ap=argparse.ArgumentParser(description=__doc__) + ap.add_argument('--local',action='store_true',help='Use installed toolchain; still compile Android only') + ap.add_argument('--cpu',choices=CPUS,default='aarch64') + ap.add_argument('--profiles',default='all',help='all or comma-separated IDs from config/profiles.json') + ap.add_argument('--no-fetch',action='store_true',help='Require previously fetched/verified source tree') + ap.add_argument('--assets',help='Optional private KaM resource root; no resources are downloaded') + ap.add_argument('--asset-path',action='append',help='Relative path under --assets to stage. Repeat for a small selected fixture set.') + ap.add_argument('--package-all',action='store_true',help='Package every successfully linked probe for device startup checks') + ap.add_argument('--timeout',type=int,default=1800,help='Per-stage safety timeout in seconds') + a=ap.parse_args() + if a.asset_path and not a.assets:ap.error('--asset-path requires --assets') + if a.timeout<1:ap.error('--timeout must be positive') + try:profiles=selected_profiles(a.profiles) + except ProbeError as e:ap.error(str(e)) + return build_local(a,profiles) if a.local else docker_build(a,profiles) + +if __name__=='__main__':raise SystemExit(main()) diff --git a/tools/source_audit.py b/tools/source_audit.py new file mode 100755 index 0000000..114bd07 --- /dev/null +++ b/tools/source_audit.py @@ -0,0 +1,40 @@ +"""Lexical dependency hints, explicitly not a substitute for Android compilation.""" +from __future__ import annotations +import re +from pathlib import Path +from common import sha256 + +RISKS={ + 'desktop_ui':r'\b(?:Forms|Controls|Graphics|Dialogs|LCLIntf|LCLType|Interfaces)\b', + 'windows_api':r'\b(?:Windows|Winapi|ShellAPI|MMSystem|Messages)\b', + 'legacy_gl':r'\b(?:dglOpenGL|glBegin|glEnd|glMatrixMode|glVertexPointer|glOrtho)\b', + 'native_audio':r'\b(?:BASS\w*|OpenAL|DirectSound)\b', + 'native_files':r'\b(?:TFileStream|FileExists|FindFirst|FindNext|GetCurrentDir|ParamStr)\b', + 'layout_or_assembler':r'\b(?:Extended|NativeUInt|NativeInt|Pointer|assembler|asm|packed\s+record)\b', + 'initialization':r'^\s*initialization\b'} + +def inventory(source:Path)->dict: + units={};risks=[];edges=[];files=[] + for p in sorted(source.rglob('*')): + if not p.is_file() or p.suffix.lower() not in ('.pas','.pp'):continue + text=p.read_text(encoding='utf-8-sig',errors='replace');rel=p.relative_to(source).as_posix() + m=re.search(r'^\s*unit\s+([\w.]+)\s*;',text,re.I|re.M) + if not m:continue + units.setdefault(m[1].casefold(),[]).append(rel) + files.append({'path':rel,'sha256':sha256(p)}) + clean=re.sub(r'\{.*?\}|\(\*.*?\*\)|//[^\n]*',' ',text,flags=re.S) + for use in re.finditer(r'\buses\s+(.+?);',clean,re.I|re.S): + for term in use[1].split(','): + dep=re.match(r'\s*([\w.]+)',term) + if dep:edges.append({'from':m[1],'to':dep[1]}) + for kind,pattern in RISKS.items(): + for hit in list(re.finditer(pattern,text,re.I|re.M))[:8]: + risks.append({'path':rel,'line':text[:hit.start()].count('\n')+1,'kind':kind,'token':hit[0]}) + return {'scope':'Lexical, both conditional branches included; compilation decides active blockers.', + 'units':units,'duplicates':{k:v for k,v in units.items() if len(v)>1},'edges':edges,'risks':risks,'files':files} + +def search_paths(source:Path)->list[Path]: + dirs={source} + for p in source.rglob('*'): + if p.is_file() and p.suffix.lower() in ('.pas','.pp','.inc','.inl'):dirs.add(p.parent) + return sorted(dirs,key=lambda p:('ext' in p.relative_to(source).parts,len(p.parts),str(p))) diff --git a/tools/source_fetch.py b/tools/source_fetch.py new file mode 100755 index 0000000..47cf147 --- /dev/null +++ b/tools/source_fetch.py @@ -0,0 +1,147 @@ +"""Fetch only source-text blobs at r16020. Never clone or download release archives. +On the server, downloads Pascal/includes and dependency source, NOT data/maps/sprites, +DLLs, .so, .a, .o, compiled resources, executables, or history. Requires Python stdlib. +""" +from __future__ import annotations +import argparse, concurrent.futures, hashlib, json, os, re, time, urllib.error, urllib.parse, urllib.request +from pathlib import Path, PurePosixPath +from common import ROOT, COMMIT, REPOSITORY, ProbeError, git_blob, safe_relative, write_json, sha256, stamp + +CODE = {'.pas', '.pp', '.inc', '.inl', '.h', '.hpp', '.c', '.cpp', '.s', '.asm'} +SKIP_PARTS = {'examples','example','demos','demo','tests','test','docs','doc','.git'} +MAX_BLOB = 16 * 1024 * 1024 +SUBMODULES = {'src/ext/pascalscript_submodule': 'reyandme/pascalscript'} + +def selected(path: str, main: bool = True) -> bool: + safe_relative(path) + p = PurePosixPath(path) + if any(x.casefold() in SKIP_PARTS for x in p.parts[:-1]): return False + license_name = p.name.casefold().startswith(('license','licence','copying','copyright')) + if license_name: return p.suffix.casefold() in ('', '.txt', '.md', '.rst') + if p.suffix.casefold() not in CODE: return False + return not main or path.startswith('src/') or '/' not in path + +def request(url: str, max_bytes: int = 32*1024*1024) -> bytes: + headers = {'User-Agent': 'KaM-Android-Architecture-Probe/0.3', 'Accept':'application/vnd.github+json'} + # Never send a token to the raw-content host, never print tokens. + token = os.environ.get('GITHUB_TOKEN') + if token and urllib.parse.urlparse(url).hostname == 'api.github.com': + headers['Authorization'] = 'Bearer ' + token + error = None + for attempt in range(3): + try: + with urllib.request.urlopen(urllib.request.Request(url, headers=headers), timeout=60) as r: + result = r.read(max_bytes+1) + if len(result) > max_bytes: raise ProbeError('Oversized source response: '+url) + return result + except (urllib.error.URLError, TimeoutError) as e: + error = e + if isinstance(e, urllib.error.HTTPError) and e.code in (401,403,404): break + time.sleep(attempt+1) + raise ProbeError('Source request failed: '+url+'; '+str(error)) + +def remote_tree(repo: str, ref: str) -> dict: + if not re.fullmatch(r'[a-f0-9]{40}', ref): raise ProbeError('A full pinned commit is required') + result = json.loads(request(f'https://api.github.com/repos/{repo}/git/trees/{ref}?recursive=1')) + if result.get('truncated'): raise ProbeError('GitHub returned truncated tree; refusing incomplete source set') + if not isinstance(result.get('tree'), list): raise ProbeError('Missing git tree') + return result + +def enumerate_blobs(tree: dict, repo: str, ref: str, prefix: str = '', main: bool = True) -> list[dict]: + result = [] + for entry in tree['tree']: + name = safe_relative(entry['path']) + if entry['type'] != 'blob' or entry.get('mode') not in ('100644','100755'): continue + if not selected(name, main): continue + if not isinstance(entry.get('size'), int) or not 0 <= entry['size'] <= MAX_BLOB: + raise ProbeError('Unsupported source size: '+name) + result.append({'path':prefix+name, 'remote_path':name, 'repository':repo, + 'commit':ref, 'git_blob_sha1':entry['sha'], 'size':entry['size']}) + return result + +def fetch(destination: Path) -> dict: + destination.mkdir(parents=True, exist_ok=True) + tree = remote_tree(REPOSITORY, COMMIT) + write_json(destination/'upstream-tree.json', tree) + entries = enumerate_blobs(tree, REPOSITORY, COMMIT) + modules=[] + for item in tree['tree']: + if item['type'] != 'commit' or not item['path'].startswith('src/'): continue + name=safe_relative(item['path']); repo=SUBMODULES.get(name) + if not repo: raise ProbeError('Unconfigured source submodule: '+name) + child=remote_tree(repo,item['sha']) + modules.append({'path':name,'repository':repo,'commit':item['sha']}) + entries.extend(enumerate_blobs(child,repo,item['sha'],name+'/',False)) + names=[e['path'] for e in entries] + if len(names)!=len(set(names)): raise ProbeError('Overlapping source paths') + if not entries: raise ProbeError('No source blobs selected') + def obtain(e: dict) -> dict: + target=destination/e['path'];target.parent.mkdir(parents=True,exist_ok=True) + if target.is_symlink(): raise ProbeError('Symlink in source tree: '+str(target)) + if target.exists(): + data=target.read_bytes() + if git_blob(data)!=e['git_blob_sha1']: + raise ProbeError('Modified or corrupt baseline: '+str(target)+'; move it aside before refetch') + else: + rawpath=urllib.parse.quote(e['remote_path'],safe='/') + data=request(f"https://raw.githubusercontent.com/{e['repository']}/{e['commit']}/{rawpath}",MAX_BLOB) + if b'\0' in data: raise ProbeError('Non-text source blob rejected: '+e['path']) + if len(data)!=e['size'] or git_blob(data)!=e['git_blob_sha1']: + raise ProbeError('Source identity mismatch: '+e['path']) + temp=target.with_name(target.name+'.download');temp.write_bytes(data);os.replace(temp,target) + return {**e,'sha256':hashlib.sha256(data).hexdigest()} + done=[] + with concurrent.futures.ThreadPoolExecutor(max_workers=6) as pool: + for e in pool.map(obtain,entries): + done.append(e) + if len(done)%100==0: print(f'Source only: {len(done)}/{len(entries)} files verified',flush=True) + receipt={'schema':'kam-source-receipt/1','commit':COMMIT,'repository':REPOSITORY, + 'downloaded_at':stamp(),'source_only':True,'submodules':modules, + 'files':sorted(done,key=lambda e:e['path'])} + write_json(destination/'SOURCE_RECEIPT.json',receipt) + verify(destination) + return receipt + +def verify(destination: Path) -> dict: + file=destination/'SOURCE_RECEIPT.json' + if not file.is_file(): raise ProbeError('Source receipt missing; run tools/source_fetch.py') + d=json.loads(file.read_text()) + if d.get('commit')!=COMMIT or d.get('repository')!=REPOSITORY or not d.get('source_only'): + raise ProbeError('Wrong source baseline') + expected=set() + for item in d.get('files',[]): + name=safe_relative(item['path']);p=destination/name + if name in expected: raise ProbeError('Duplicate source receipt entry') + expected.add(name) + if p.is_symlink() or not p.is_file(): raise ProbeError('Source missing/linked: '+name) + if p.stat().st_size!=item['size'] or sha256(p)!=item['sha256'] or git_blob(p.read_bytes())!=item['git_blob_sha1']: + raise ProbeError('Source modified: '+name) + if not expected: raise ProbeError('Empty source set') + for p in destination.rglob('*'): + if p.is_symlink(): raise ProbeError('Symlink in source baseline: '+str(p)) + if p.is_file() and p.suffix.lower() in (CODE|{'.ppu','.o','.a','.so','.dll','.res'}): + if p.relative_to(destination).as_posix() not in expected: + raise ProbeError('Unrecorded source/binary in baseline: '+str(p)) + return d + +def import_local(source: Path, destination: Path) -> dict: + """Import an ALREADY pinned code-only receipt tree; no arbitrary git tree trusted.""" + import shutil + d=verify(source) + if destination.exists() and any(destination.iterdir()): raise ProbeError('Destination must be empty') + destination.mkdir(parents=True,exist_ok=True) + for item in d['files']: + p=destination/item['path'];p.parent.mkdir(parents=True,exist_ok=True) + shutil.copy2(source/item['path'],p) + write_json(destination/'SOURCE_RECEIPT.json',d) + return verify(destination) + +if __name__=='__main__': + ap=argparse.ArgumentParser(description=__doc__) + ap.add_argument('--verify',action='store_true');ap.add_argument('--destination',type=Path,default=ROOT/'vendor/kam_remake') + a=ap.parse_args() + try: + d=verify(a.destination) if a.verify else fetch(a.destination) + print(f"Verified {len(d['files'])} source-text files, commit {COMMIT}") + except (ProbeError,OSError,ValueError,KeyError) as e: + print('SOURCE BLOCKED:',e);raise SystemExit(2)