1260 lines
64 KiB
PowerShell
1260 lines
64 KiB
PowerShell
#requires -Version 5.1
|
|
[CmdletBinding()]
|
|
param(
|
|
[ValidateSet('Inventory', 'Migrate', 'Integrate', 'Build', 'Replay', 'Recent', 'All')]
|
|
[string]$Phase = 'Inventory',
|
|
[string]$ManifestPath = '',
|
|
[string]$BotPath = '',
|
|
[double]$FloatTolerance = 1e-9,
|
|
[ValidateRange(0, 10)]
|
|
[int]$MaxRepairAttempts = 3,
|
|
[ValidateRange(0, 5)]
|
|
[int]$MaxCodexValidationAttempts = 2,
|
|
[ValidateSet('low', 'medium', 'high')]
|
|
[string]$CodexReasoning = 'medium',
|
|
[string]$DatabaseUrl = $env:BANBOT_DATABASE_URL,
|
|
[string]$DatabaseOverlay = '',
|
|
[ValidateSet('', 'timescale', 'questdb')]
|
|
[string]$DatabaseType = '',
|
|
[ValidateRange(1, 1440)]
|
|
[int]$BacktestTimeoutMinutes = 120,
|
|
[switch]$Force
|
|
)
|
|
|
|
Set-StrictMode -Version Latest
|
|
$ErrorActionPreference = 'Stop'
|
|
|
|
$script:RepoRoot = [IO.Path]::GetFullPath((Join-Path $PSScriptRoot '..'))
|
|
if (-not $ManifestPath) { $ManifestPath = Join-Path $script:RepoRoot 'code\migration_manifest.json' }
|
|
if (-not $BotPath) { $BotPath = Join-Path $script:RepoRoot 'bot.exe' }
|
|
$script:ManifestPath = [IO.Path]::GetFullPath($ManifestPath)
|
|
$script:Manifest = Get-Content -LiteralPath $script:ManifestPath -Raw | ConvertFrom-Json
|
|
$script:StatePath = [IO.Path]::GetFullPath((Join-Path $script:RepoRoot $script:Manifest.stateFile))
|
|
$script:SummaryPath = [IO.Path]::GetFullPath((Join-Path $script:RepoRoot $script:Manifest.summaryFile))
|
|
$script:WorkRoot = Join-Path $script:RepoRoot 'code\.migration-work'
|
|
$script:LogRoot = Join-Path $script:RepoRoot 'code\migration-logs'
|
|
$script:RecentToday = [DateTime]::UtcNow.Date
|
|
$script:AllReportPolicies = @()
|
|
$script:UncoveredPolicies = @()
|
|
|
|
function Write-JsonAtomic {
|
|
param([Parameter(Mandatory)]$Value, [Parameter(Mandatory)][string]$Path)
|
|
$parent = Split-Path -Parent $Path
|
|
New-Item -ItemType Directory -Force -Path $parent | Out-Null
|
|
$temp = "$Path.tmp"
|
|
$Value | ConvertTo-Json -Depth 100 | Set-Content -LiteralPath $temp -Encoding UTF8
|
|
Move-Item -LiteralPath $temp -Destination $Path -Force
|
|
}
|
|
|
|
function Get-RelativePath {
|
|
param([Parameter(Mandatory)][string]$Base, [Parameter(Mandatory)][string]$Path)
|
|
$baseFull = [IO.Path]::GetFullPath($Base).TrimEnd('\', '/') + [IO.Path]::DirectorySeparatorChar
|
|
$pathFull = [IO.Path]::GetFullPath($Path)
|
|
$baseUri = New-Object Uri($baseFull)
|
|
$pathUri = New-Object Uri($pathFull)
|
|
return [Uri]::UnescapeDataString($baseUri.MakeRelativeUri($pathUri).ToString()).Replace('/', [IO.Path]::DirectorySeparatorChar)
|
|
}
|
|
|
|
function Assert-DescendantPath {
|
|
param([Parameter(Mandatory)][string]$Root, [Parameter(Mandatory)][string]$Path)
|
|
$rootFull = [IO.Path]::GetFullPath($Root).TrimEnd('\', '/') + [IO.Path]::DirectorySeparatorChar
|
|
$pathFull = [IO.Path]::GetFullPath($Path)
|
|
if (-not $pathFull.StartsWith($rootFull, [StringComparison]::OrdinalIgnoreCase)) {
|
|
throw "Unsafe path outside expected root: $pathFull"
|
|
}
|
|
return $pathFull
|
|
}
|
|
|
|
function Assert-NoReparsePath {
|
|
param([Parameter(Mandatory)][string]$Root, [Parameter(Mandatory)][string]$Path)
|
|
$rootFull = [IO.Path]::GetFullPath($Root).TrimEnd('\', '/')
|
|
$pathFull = Assert-DescendantPath -Root $rootFull -Path $Path
|
|
$relative = $pathFull.Substring($rootFull.Length).TrimStart('\', '/')
|
|
$current = $rootFull
|
|
foreach ($part in @($relative -split '[\\/]' | Where-Object { $_ })) {
|
|
$current = Join-Path $current $part
|
|
if (Test-Path -LiteralPath $current) {
|
|
$item = Get-Item -LiteralPath $current -Force
|
|
if (($item.Attributes -band [IO.FileAttributes]::ReparsePoint) -ne 0) {
|
|
throw "Refusing reparse-point path: $current"
|
|
}
|
|
}
|
|
}
|
|
return $pathFull
|
|
}
|
|
|
|
function Read-State {
|
|
if (Test-Path -LiteralPath $script:StatePath) {
|
|
return Get-Content -LiteralPath $script:StatePath -Raw | ConvertFrom-Json
|
|
}
|
|
return [pscustomobject]@{
|
|
schemaVersion = 1
|
|
manifest = $script:ManifestPath
|
|
completed = @()
|
|
updatedUtc = [DateTime]::UtcNow.ToString('o')
|
|
}
|
|
}
|
|
|
|
function Test-Completed {
|
|
param([Parameter(Mandatory)][string]$Key)
|
|
$state = Read-State
|
|
return @($state.completed) -contains $Key
|
|
}
|
|
|
|
function Set-Completed {
|
|
param([Parameter(Mandatory)][string]$Key)
|
|
$state = Read-State
|
|
$completed = @($state.completed)
|
|
if ($completed -notcontains $Key) {
|
|
$completed += $Key
|
|
}
|
|
$state.completed = @($completed | Sort-Object -Unique)
|
|
$state.updatedUtc = [DateTime]::UtcNow.ToString('o')
|
|
Write-JsonAtomic -Value $state -Path $script:StatePath
|
|
}
|
|
|
|
function Invoke-LoggedCommand {
|
|
param(
|
|
[Parameter(Mandatory)][string]$FilePath,
|
|
[Parameter(Mandatory)][string[]]$Arguments,
|
|
[Parameter(Mandatory)][string]$LogPath,
|
|
[string]$InputText = '',
|
|
[string]$WorkingDirectory = $script:RepoRoot,
|
|
[ValidateRange(1, 86400)][int]$TimeoutSeconds = 3600
|
|
)
|
|
New-Item -ItemType Directory -Force -Path (Split-Path -Parent $LogPath) | Out-Null
|
|
if (-not [IO.Path]::IsPathRooted($FilePath)) {
|
|
$application = Get-Command $FilePath -CommandType Application -ErrorAction SilentlyContinue |
|
|
Where-Object { $_.Source -match '\.(?:cmd|bat|com|exe)$' } |
|
|
Select-Object -First 1
|
|
if ($null -eq $application) { throw "Executable not found: $FilePath" }
|
|
$FilePath = $application.Source
|
|
}
|
|
$quotedArgs = foreach ($argument in $Arguments) {
|
|
$escaped = [regex]::Replace([string]$argument, '(\\*)"', '$1$1\"')
|
|
$escaped = [regex]::Replace($escaped, '(\\+)$', '$1$1')
|
|
'"' + $escaped + '"'
|
|
}
|
|
$startInfo = New-Object Diagnostics.ProcessStartInfo
|
|
if ([IO.Path]::GetExtension($FilePath) -match '^\.(?:cmd|bat)$') {
|
|
$startInfo.FileName = $env:ComSpec
|
|
$startInfo.Arguments = '/d /s /c ""' + $FilePath + '" ' + ($quotedArgs -join ' ') + '"'
|
|
} else {
|
|
$startInfo.FileName = $FilePath
|
|
$startInfo.Arguments = $quotedArgs -join ' '
|
|
}
|
|
$startInfo.WorkingDirectory = $WorkingDirectory
|
|
$startInfo.UseShellExecute = $false
|
|
$startInfo.CreateNoWindow = $true
|
|
$startInfo.RedirectStandardOutput = $true
|
|
$startInfo.RedirectStandardError = $true
|
|
$startInfo.RedirectStandardInput = $true
|
|
$utf8NoBom = New-Object Text.UTF8Encoding($false)
|
|
if ($startInfo.PSObject.Properties.Name -contains 'StandardInputEncoding') { $startInfo.StandardInputEncoding = $utf8NoBom }
|
|
if ($startInfo.PSObject.Properties.Name -contains 'StandardOutputEncoding') { $startInfo.StandardOutputEncoding = $utf8NoBom }
|
|
if ($startInfo.PSObject.Properties.Name -contains 'StandardErrorEncoding') { $startInfo.StandardErrorEncoding = $utf8NoBom }
|
|
$process = New-Object Diagnostics.Process
|
|
$process.StartInfo = $startInfo
|
|
try {
|
|
if (-not $process.Start()) { throw "Failed to start $FilePath" }
|
|
$stdoutTask = $process.StandardOutput.ReadToEndAsync()
|
|
$stderrTask = $process.StandardError.ReadToEndAsync()
|
|
if ($InputText) {
|
|
$inputBytes = [Text.Encoding]::UTF8.GetBytes($InputText)
|
|
$process.StandardInput.BaseStream.Write($inputBytes, 0, $inputBytes.Length)
|
|
$process.StandardInput.BaseStream.Flush()
|
|
}
|
|
$process.StandardInput.BaseStream.Close()
|
|
$timedOut = -not $process.WaitForExit($TimeoutSeconds * 1000)
|
|
if ($timedOut) {
|
|
try { $process.Kill() } catch { }
|
|
$process.WaitForExit()
|
|
}
|
|
$stdout = $stdoutTask.Result
|
|
$stderr = $stderrTask.Result
|
|
$exitCode = $process.ExitCode
|
|
$output = @($stdout, $stderr) | Where-Object { $_ }
|
|
($output -join [Environment]::NewLine) | Set-Content -LiteralPath $LogPath -Encoding UTF8
|
|
if ($timedOut) { throw "Command timed out after $TimeoutSeconds seconds: $FilePath" }
|
|
} finally {
|
|
$process.Dispose()
|
|
}
|
|
return [pscustomobject]@{ ExitCode = $exitCode; Output = @($output); LogPath = $LogPath }
|
|
}
|
|
|
|
function Get-ReportRoot {
|
|
param([Parameter(Mandatory)]$Report)
|
|
return [IO.Path]::GetFullPath((Join-Path $script:Manifest.reportsRoot $Report.id))
|
|
}
|
|
|
|
function Get-ConfigPolicyNames {
|
|
param([Parameter(Mandatory)][string]$Path)
|
|
$inside = $false
|
|
$names = New-Object Collections.Generic.List[string]
|
|
foreach ($line in Get-Content -LiteralPath $Path) {
|
|
if ($line -match '^run_policy\s*:') {
|
|
$inside = $true
|
|
continue
|
|
}
|
|
if ($inside -and $line -match '^\S') { $inside = $false }
|
|
if ($inside -and $line -match '^\s{2}-\s+name:\s*(?<name>[^#\s]+)') {
|
|
$names.Add($Matches.name)
|
|
}
|
|
}
|
|
return @($names | Sort-Object -Unique)
|
|
}
|
|
|
|
function Get-PolicyParts {
|
|
param([Parameter(Mandatory)][string]$Name)
|
|
$parts = $Name.Split(':', 2)
|
|
if ($parts.Count -eq 1) {
|
|
return [pscustomobject]@{ Group = $Name; Strategy = $Name }
|
|
}
|
|
return [pscustomobject]@{ Group = $parts[0]; Strategy = $parts[1] }
|
|
}
|
|
|
|
function Get-TargetSourceFiles {
|
|
param([Parameter(Mandatory)][string]$ReportRoot, [Parameter(Mandatory)][string[]]$Policies)
|
|
$files = New-Object Collections.Generic.List[IO.FileInfo]
|
|
foreach ($group in @($Policies | ForEach-Object { (Get-PolicyParts $_).Group } | Sort-Object -Unique)) {
|
|
$groupDir = Join-Path $ReportRoot "strat_$group"
|
|
if (-not (Test-Path -LiteralPath $groupDir -PathType Container)) { return @() }
|
|
if ($group -eq 'llmjob') {
|
|
$main = Join-Path $groupDir 'main.go'
|
|
if (Test-Path -LiteralPath $main -PathType Leaf) { $files.Add((Get-Item -LiteralPath $main)) }
|
|
foreach ($policy in @($Policies | Where-Object { (Get-PolicyParts $_).Group -eq $group })) {
|
|
$strategy = (Get-PolicyParts $policy).Strategy
|
|
$relative = if ($strategy.StartsWith('H', [StringComparison]::Ordinal) -and $strategy.Length -gt 1) {
|
|
Join-Path 'his' ($strategy.Substring(1) + '.go')
|
|
} else {
|
|
$strategy + '.go'
|
|
}
|
|
$source = Join-Path $groupDir $relative
|
|
if (-not (Test-Path -LiteralPath $source -PathType Leaf)) { return @() }
|
|
$files.Add((Get-Item -LiteralPath $source))
|
|
}
|
|
} else {
|
|
foreach ($file in Get-ChildItem -LiteralPath $groupDir -File -Filter '*.go' -Recurse | Where-Object Name -notlike '*_test.go') {
|
|
$files.Add($file)
|
|
}
|
|
}
|
|
}
|
|
return @($files | Sort-Object FullName -Unique | ForEach-Object {
|
|
(Get-RelativePath -Base $ReportRoot -Path $_.FullName).Replace('\', '/')
|
|
})
|
|
}
|
|
|
|
function New-AutoRepresentative {
|
|
param([Parameter(Mandatory)][IO.DirectoryInfo]$Directory, [Parameter(Mandatory)][string[]]$Policies)
|
|
$sources = @(Get-TargetSourceFiles -ReportRoot $Directory.FullName -Policies $Policies)
|
|
if ($sources.Count -eq 0) { return $null }
|
|
$groupName = "migrated_$($Directory.Name)"
|
|
$policyMap = [ordered]@{}
|
|
foreach ($policy in $Policies) {
|
|
$policyMap[$policy] = "${groupName}:$((Get-PolicyParts $policy).Strategy)"
|
|
}
|
|
$target = [ordered]@{
|
|
id = $Directory.Name
|
|
sourceFiles = $sources
|
|
generatedGroup = $groupName
|
|
policyMap = [pscustomobject]$policyMap
|
|
}
|
|
$configPath = Join-Path $Directory.FullName 'config.yml'
|
|
if (Select-String -LiteralPath $configPath -Pattern '^\s+use_latest\s*:\s*true\s*(?:#.*)?$' -Quiet) {
|
|
$target.dynamicUniverse = [pscustomobject]@{ source = 'pairmgr/use_latest'; sourceLog = 'out.log' }
|
|
}
|
|
return [pscustomobject]$target
|
|
}
|
|
|
|
function Resolve-Representatives {
|
|
$resolved = New-Object Collections.Generic.List[object]
|
|
$covered = New-Object 'Collections.Generic.HashSet[string]' ([StringComparer]::Ordinal)
|
|
foreach ($report in @($script:Manifest.representatives)) {
|
|
$resolved.Add($report)
|
|
foreach ($policy in $report.policyMap.psobject.Properties.Name) { $null = $covered.Add($policy) }
|
|
}
|
|
|
|
$root = [IO.Path]::GetFullPath($script:Manifest.reportsRoot)
|
|
$candidates = New-Object Collections.Generic.List[object]
|
|
$allPolicies = New-Object 'Collections.Generic.HashSet[string]' ([StringComparer]::Ordinal)
|
|
foreach ($directory in Get-ChildItem -LiteralPath $root -Directory | Sort-Object Name) {
|
|
$configPath = Join-Path $directory.FullName 'config.yml'
|
|
$detailPath = Join-Path $directory.FullName 'detail.json'
|
|
if (-not (Test-Path -LiteralPath $configPath -PathType Leaf) -or -not (Test-Path -LiteralPath $detailPath -PathType Leaf)) { continue }
|
|
$policies = @(Get-ConfigPolicyNames -Path $configPath)
|
|
if ($policies.Count -eq 0) { continue }
|
|
foreach ($policy in $policies) { $null = $allPolicies.Add($policy) }
|
|
if (@($script:Manifest.representatives.id) -contains $directory.Name) { continue }
|
|
$candidate = New-AutoRepresentative -Directory $directory -Policies $policies
|
|
if ($null -ne $candidate) { $candidates.Add($candidate) }
|
|
}
|
|
|
|
while (@($allPolicies | Where-Object { -not $covered.Contains($_) }).Count -gt 0) {
|
|
$best = $null
|
|
$bestNew = @()
|
|
foreach ($candidate in $candidates) {
|
|
$newPolicies = @($candidate.policyMap.psobject.Properties.Name | Where-Object { -not $covered.Contains($_) })
|
|
if ($newPolicies.Count -gt $bestNew.Count) {
|
|
$best = $candidate
|
|
$bestNew = $newPolicies
|
|
}
|
|
}
|
|
if ($null -eq $best -or $bestNew.Count -eq 0) {
|
|
$missing = @($allPolicies | Where-Object { -not $covered.Contains($_) } | Sort-Object)
|
|
$script:UncoveredPolicies = $missing
|
|
Write-Warning "Skipping policies with no report source snapshot: $($missing -join ', ')"
|
|
break
|
|
}
|
|
$resolved.Add($best)
|
|
foreach ($policy in $bestNew) { $null = $covered.Add($policy) }
|
|
$remaining = @($candidates | Where-Object id -ne $best.id)
|
|
$candidates = New-Object Collections.Generic.List[object]
|
|
foreach ($candidate in $remaining) { $candidates.Add($candidate) }
|
|
}
|
|
$script:AllReportPolicies = @($allPolicies | Sort-Object)
|
|
if ($null -eq $script:UncoveredPolicies) { $script:UncoveredPolicies = @() }
|
|
return @($resolved | ForEach-Object { $_ })
|
|
}
|
|
|
|
function Assert-RepresentativeInputs {
|
|
param([Parameter(Mandatory)]$Report)
|
|
$reportRoot = Get-ReportRoot $Report
|
|
foreach ($required in @('config.yml', 'detail.json')) {
|
|
$path = Join-Path $reportRoot $required
|
|
if (-not (Test-Path -LiteralPath $path -PathType Leaf)) {
|
|
throw "Missing $required for report $($Report.id): $path"
|
|
}
|
|
}
|
|
foreach ($source in @($Report.sourceFiles)) {
|
|
$path = Join-Path $reportRoot $source
|
|
Assert-DescendantPath -Root $reportRoot -Path $path | Out-Null
|
|
if (-not (Test-Path -LiteralPath $path -PathType Leaf)) {
|
|
throw "Missing allowlisted source for report $($Report.id): $source"
|
|
}
|
|
}
|
|
}
|
|
|
|
function Get-SourceFingerprint {
|
|
param([Parameter(Mandatory)][string]$ReportRoot, [Parameter(Mandatory)][AllowEmptyCollection()][IO.FileInfo[]]$Files)
|
|
if ($Files.Count -eq 0) { return $null }
|
|
$rows = foreach ($file in $Files) {
|
|
$relative = (Get-RelativePath -Base $ReportRoot -Path $file.FullName).Replace('\', '/')
|
|
"$relative|$((Get-FileHash -Algorithm SHA256 -LiteralPath $file.FullName).Hash)"
|
|
}
|
|
$payload = [Text.Encoding]::UTF8.GetBytes((($rows | Sort-Object) -join "`n"))
|
|
$sha = [Security.Cryptography.SHA256]::Create()
|
|
try { return ([BitConverter]::ToString($sha.ComputeHash($payload))).Replace('-', '').ToLowerInvariant() }
|
|
finally { $sha.Dispose() }
|
|
}
|
|
|
|
function Invoke-Inventory {
|
|
$root = [IO.Path]::GetFullPath($script:Manifest.reportsRoot)
|
|
$reportDirs = @(Get-ChildItem -LiteralPath $root -Directory)
|
|
$allDirs = @(Get-ChildItem -LiteralPath $root -Directory -Recurse)
|
|
$configFiles = @(Get-ChildItem -LiteralPath $root -File -Filter 'config.yml' -Recurse)
|
|
$detailFiles = @(Get-ChildItem -LiteralPath $root -File -Filter 'detail.json' -Recurse)
|
|
$sourceFiles = @(Get-ChildItem -LiteralPath $root -File -Filter '*.go' -Recurse | Where-Object {
|
|
$_.FullName -match '[\\/]strat_[^\\/]+[\\/]'
|
|
})
|
|
|
|
$reports = foreach ($dir in $reportDirs) {
|
|
$sources = @(Get-ChildItem -LiteralPath $dir.FullName -File -Filter '*.go' -Recurse | Where-Object {
|
|
$_.FullName -match '[\\/]strat_[^\\/]+[\\/]'
|
|
})
|
|
[pscustomobject]@{
|
|
id = $dir.Name
|
|
configCount = @(Get-ChildItem -LiteralPath $dir.FullName -File -Filter 'config.yml' -Recurse).Count
|
|
detailCount = @(Get-ChildItem -LiteralPath $dir.FullName -File -Filter 'detail.json' -Recurse).Count
|
|
sourceCount = $sources.Count
|
|
sourceFingerprint = Get-SourceFingerprint -ReportRoot $dir.FullName -Files $sources
|
|
}
|
|
}
|
|
|
|
$duplicates = @($reports | Where-Object sourceFingerprint | Group-Object sourceFingerprint | Where-Object Count -gt 1 | ForEach-Object {
|
|
[pscustomobject]@{
|
|
fingerprint = $_.Name
|
|
count = $_.Count
|
|
reports = @($_.Group.id | Sort-Object)
|
|
}
|
|
})
|
|
$inventory = [pscustomobject]@{
|
|
schemaVersion = 1
|
|
generatedUtc = [DateTime]::UtcNow.ToString('o')
|
|
root = $root
|
|
counts = [pscustomobject]@{
|
|
reportDirectories = $reportDirs.Count
|
|
allDirectories = $allDirs.Count
|
|
configFiles = $configFiles.Count
|
|
detailFiles = $detailFiles.Count
|
|
strategySourceFiles = $sourceFiles.Count
|
|
duplicateFingerprints = $duplicates.Count
|
|
}
|
|
policyCoverage = [pscustomobject]@{
|
|
uniquePolicies = $script:AllReportPolicies.Count
|
|
coveredPolicies = @($script:Representatives | ForEach-Object {
|
|
$_.policyMap.psobject.Properties.Name
|
|
} | Sort-Object -Unique).Count
|
|
missingSourcePolicies = @($script:UncoveredPolicies)
|
|
representativeReports = $script:Representatives.Count
|
|
}
|
|
representatives = @($script:Representatives.id)
|
|
migrationTargets = @($script:Representatives | ForEach-Object {
|
|
[pscustomobject]@{
|
|
id = $_.id
|
|
policies = @($_.policyMap.psobject.Properties.Name)
|
|
generatedGroup = $_.generatedGroup
|
|
sourceFiles = @($_.sourceFiles)
|
|
dynamicUniverse = if ($_.psobject.Properties['dynamicUniverse']) { $_.dynamicUniverse } else { $null }
|
|
}
|
|
})
|
|
duplicateFingerprints = $duplicates
|
|
reports = @($reports)
|
|
}
|
|
$path = [IO.Path]::GetFullPath((Join-Path $script:RepoRoot $script:Manifest.inventoryFile))
|
|
Write-JsonAtomic -Value $inventory -Path $path
|
|
Set-Completed 'inventory'
|
|
Write-Host "Inventory written: $path"
|
|
Write-Host "Reports=$($reportDirs.Count) configs=$($configFiles.Count) details=$($detailFiles.Count) sources=$($sourceFiles.Count) duplicateSets=$($duplicates.Count)"
|
|
}
|
|
|
|
function Copy-MigrationInputs {
|
|
param([Parameter(Mandatory)]$Report, [Parameter(Mandatory)][string]$Scratch)
|
|
if (Test-Path -LiteralPath $Scratch) {
|
|
Assert-DescendantPath -Root $script:WorkRoot -Path $Scratch | Out-Null
|
|
Remove-Item -LiteralPath $Scratch -Recurse -Force
|
|
}
|
|
New-Item -ItemType Directory -Force -Path (Join-Path $Scratch 'input'), (Join-Path $Scratch 'output') | Out-Null
|
|
Copy-Item -LiteralPath $script:Manifest.knowledgeFile -Destination (Join-Path $Scratch 'input\banbot_cn.md')
|
|
$reportRoot = Get-ReportRoot $Report
|
|
foreach ($relative in @($Report.sourceFiles)) {
|
|
$destination = Join-Path (Join-Path $Scratch 'input\sources') $relative
|
|
New-Item -ItemType Directory -Force -Path (Split-Path -Parent $destination) | Out-Null
|
|
Copy-Item -LiteralPath (Join-Path $reportRoot $relative) -Destination $destination
|
|
}
|
|
}
|
|
|
|
function Get-PolicyInstructions {
|
|
param([Parameter(Mandatory)]$Report)
|
|
return @($Report.policyMap.psobject.Properties | ForEach-Object { "- $($_.Name) -> $($_.Value)" }) -join "`n"
|
|
}
|
|
|
|
function Get-InlineFileBlock {
|
|
param([Parameter(Mandatory)][string]$Label, [Parameter(Mandatory)][string]$Path)
|
|
$content = Get-Content -LiteralPath $Path -Raw
|
|
return "<<<FILE $Label>>>`n$content`n<<<END FILE>>>"
|
|
}
|
|
|
|
function Write-CodexFilesSchema {
|
|
param([Parameter(Mandatory)][string]$Path)
|
|
$schema = @'
|
|
{
|
|
"type": "object",
|
|
"properties": {
|
|
"files": {
|
|
"type": "array",
|
|
"items": {
|
|
"type": "object",
|
|
"properties": {
|
|
"path": { "type": "string" },
|
|
"content": { "type": "string" }
|
|
},
|
|
"required": ["path", "content"],
|
|
"additionalProperties": false
|
|
}
|
|
}
|
|
},
|
|
"required": ["files"],
|
|
"additionalProperties": false
|
|
}
|
|
'@
|
|
[IO.File]::WriteAllText($Path, $schema, (New-Object Text.UTF8Encoding($false)))
|
|
}
|
|
|
|
function Write-CodexResponseFiles {
|
|
param(
|
|
[Parameter(Mandatory)][string]$ResponsePath,
|
|
[Parameter(Mandatory)][string]$Destination
|
|
)
|
|
try { $response = Get-Content -LiteralPath $ResponsePath -Raw | ConvertFrom-Json }
|
|
catch { throw "Codex response is not valid JSON: ${ResponsePath}: $($_.Exception.Message)" }
|
|
$files = @($response.files)
|
|
if ($files.Count -eq 0) { throw "Codex response contains no files: $ResponsePath" }
|
|
New-Item -ItemType Directory -Force -Path $Destination | Out-Null
|
|
foreach ($file in $files) {
|
|
$relative = ([string]$file.path).Replace('/', [IO.Path]::DirectorySeparatorChar)
|
|
if ([string]::IsNullOrWhiteSpace($relative) -or [IO.Path]::IsPathRooted($relative) -or
|
|
$relative -match '(^|[\\/])\.\.([\\/]|$)' -or [IO.Path]::GetExtension($relative) -ne '.go') {
|
|
throw "Unsafe or non-Go path in Codex response: $relative"
|
|
}
|
|
$target = [IO.Path]::GetFullPath((Join-Path $Destination $relative))
|
|
Assert-DescendantPath -Root $Destination -Path $target | Out-Null
|
|
New-Item -ItemType Directory -Force -Path (Split-Path -Parent $target) | Out-Null
|
|
Set-Content -LiteralPath $target -Value ([string]$file.content) -Encoding UTF8
|
|
}
|
|
}
|
|
|
|
function Assert-GeneratedPackage {
|
|
param([Parameter(Mandatory)]$Report, [Parameter(Mandatory)][string]$Directory)
|
|
$files = @(Get-ChildItem -LiteralPath $Directory -File -Filter '*.go' -Recurse)
|
|
if ($files.Count -eq 0) { throw "Codex produced no Go files for $($Report.id)" }
|
|
$text = ($files | ForEach-Object { Get-Content -LiteralPath $_.FullName -Raw }) -join "`n"
|
|
if ($text -match '\bOn(?:Info)?Bar\s*:') {
|
|
throw "Generated package still uses deprecated OnBar/OnInfoBar callbacks for $($Report.id)"
|
|
}
|
|
if ($text -notmatch '\bOnData\s*:') {
|
|
throw "Generated package does not implement OnData for $($Report.id)"
|
|
}
|
|
if ($Report.psobject.Properties['requiresRouteData'] -and $Report.requiresRouteData -and $text -notmatch '\bRouteData\s*\(') {
|
|
throw "Generated package must use RouteData for mixed main/info callbacks in $($Report.id)"
|
|
}
|
|
if ($text -notmatch [regex]::Escape("AddStratGroup(`"$($Report.generatedGroup)`"")) {
|
|
throw "Generated package does not register group $($Report.generatedGroup)"
|
|
}
|
|
foreach ($policy in $Report.policyMap.psobject.Properties) {
|
|
$strategyName = $policy.Value.Split(':', 2)[1]
|
|
if ($text -notmatch ('["'']' + [regex]::Escape($strategyName) + '["'']\s*:')) {
|
|
throw "Generated package does not register strategy key $strategyName"
|
|
}
|
|
}
|
|
$directoryFull = [IO.Path]::GetFullPath($Directory)
|
|
Assert-DescendantPath -Root $script:RepoRoot -Path $directoryFull | Out-Null
|
|
$relative = (Get-RelativePath -Base $script:RepoRoot -Path $directoryFull).Replace('\', '/')
|
|
$compile = Invoke-LoggedCommand -FilePath 'go' -Arguments @('test', "./$relative") -LogPath (Join-Path $script:LogRoot "go-test-generated-$($Report.id).log") -TimeoutSeconds 300
|
|
if ($compile.ExitCode -ne 0) {
|
|
throw "Generated package does not compile for $($Report.id); see $($compile.LogPath)"
|
|
}
|
|
}
|
|
|
|
function Invoke-CodexMigration {
|
|
param([Parameter(Mandatory)]$Report)
|
|
Assert-RepresentativeInputs $Report
|
|
$scratch = Join-Path $script:WorkRoot $Report.id
|
|
Copy-MigrationInputs -Report $Report -Scratch $scratch
|
|
$blocks = New-Object Collections.Generic.List[string]
|
|
$blocks.Add((Get-InlineFileBlock -Label 'banbot_cn.md' -Path (Join-Path $scratch 'input\banbot_cn.md')))
|
|
foreach ($relative in @($Report.sourceFiles)) {
|
|
$blocks.Add((Get-InlineFileBlock -Label $relative.Replace('\', '/') -Path (Join-Path (Join-Path $scratch 'input\sources') $relative)))
|
|
}
|
|
$inlineInputs = $blocks -join "`n`n"
|
|
$prompt = @"
|
|
Do not call tools or inspect the filesystem. All allowed inputs are included below.
|
|
Return only JSON matching the supplied schema. Each file path must be relative and end in .go.
|
|
Generate the complete compilable Go package represented by those files.
|
|
Preserve trading behavior. Convert deprecated OnBar/OnInfoBar behavior to OnData using the documented main/info-data helpers so info data cannot execute main-bar logic twice.
|
|
The package must register exactly this unique group: $($Report.generatedGroup)
|
|
Register these policy mappings:
|
|
$(Get-PolicyInstructions $Report)
|
|
|
|
Use package name report_$($Report.id). Remove registrations and code for strategies outside the mappings above. Resolve dependencies from the allowlisted files without importing the old banstagy strategy package. Do not add dependencies.
|
|
Do not include markdown fences or explanations.
|
|
|
|
$inlineInputs
|
|
"@
|
|
$logPath = Join-Path $script:LogRoot "codex-migrate-$($Report.id).log"
|
|
$schemaPath = Join-Path $scratch 'files-schema.json'
|
|
$responsePath = Join-Path $scratch 'files-response.json'
|
|
Write-CodexFilesSchema -Path $schemaPath
|
|
$reasoningConfig = 'model_reasoning_effort="' + $CodexReasoning + '"'
|
|
$result = Invoke-LoggedCommand -FilePath 'codex' -Arguments @('exec', '--disable', 'hooks', '--disable', 'multi_agent', '-c', $reasoningConfig, '--sandbox', 'read-only', '--ephemeral', '--skip-git-repo-check', '--output-schema', $schemaPath, '--output-last-message', $responsePath, '-C', $scratch, '-') -LogPath $logPath -InputText $prompt -WorkingDirectory $scratch
|
|
if ($result.ExitCode -ne 0) { throw "Codex migration failed for $($Report.id); see $logPath" }
|
|
$output = Join-Path $scratch 'output'
|
|
Write-CodexResponseFiles -ResponsePath $responsePath -Destination $output
|
|
$formatArgs = @('-w') + @(Get-ChildItem -LiteralPath $output -File -Filter '*.go' -Recurse | Select-Object -ExpandProperty FullName)
|
|
$format = Invoke-LoggedCommand -FilePath 'gofmt' -Arguments $formatArgs -LogPath (Join-Path $script:LogRoot "gofmt-migrate-$($Report.id).log")
|
|
if ($format.ExitCode -ne 0) { throw "gofmt failed for $($Report.id); see $($format.LogPath)" }
|
|
Assert-GeneratedPackage -Report $Report -Directory $output
|
|
|
|
$generatedRoot = [IO.Path]::GetFullPath((Join-Path $script:RepoRoot $script:Manifest.generatedRoot))
|
|
$destination = Join-Path $generatedRoot $Report.id
|
|
if (Test-Path -LiteralPath $destination) {
|
|
Assert-DescendantPath -Root $generatedRoot -Path $destination | Out-Null
|
|
Remove-Item -LiteralPath $destination -Recurse -Force
|
|
}
|
|
New-Item -ItemType Directory -Force -Path $destination | Out-Null
|
|
Copy-Item -Path (Join-Path $output '*') -Destination $destination -Recurse
|
|
Set-Completed "migrate:$($Report.id)"
|
|
Write-Host "Migrated $($Report.id) -> $destination"
|
|
}
|
|
|
|
function Invoke-Migrate {
|
|
foreach ($report in $script:Representatives) {
|
|
$destination = Join-Path (Join-Path $script:RepoRoot $script:Manifest.generatedRoot) $report.id
|
|
if (-not $Force -and (Test-Completed "migrate:$($report.id)") -and (Test-Path -LiteralPath $destination)) {
|
|
Write-Host "Resume: migration already complete for $($report.id)"
|
|
continue
|
|
}
|
|
Invoke-CodexMigration -Report $report
|
|
}
|
|
}
|
|
|
|
function Invoke-Integrate {
|
|
foreach ($report in $script:Representatives) {
|
|
$destination = Join-Path (Join-Path $script:RepoRoot $script:Manifest.generatedRoot) $report.id
|
|
Assert-GeneratedPackage -Report $report -Directory $destination
|
|
}
|
|
$imports = @($script:Representatives | Sort-Object id | ForEach-Object {
|
|
"`t_ `"github.com/banbox/banstagy/code/generated/$($_.id)`""
|
|
}) -join "`r`n"
|
|
$registry = @"
|
|
// Code generated by scripts/migrate_backtests.ps1; DO NOT EDIT.
|
|
package code
|
|
|
|
import (
|
|
$imports
|
|
)
|
|
"@
|
|
$registryPath = [IO.Path]::GetFullPath((Join-Path $script:RepoRoot $script:Manifest.registryFile))
|
|
New-Item -ItemType Directory -Force -Path (Split-Path -Parent $registryPath) | Out-Null
|
|
Set-Content -LiteralPath $registryPath -Value $registry -Encoding UTF8
|
|
|
|
$mainPath = Join-Path $script:RepoRoot 'main.go'
|
|
$mainText = Get-Content -LiteralPath $mainPath -Raw
|
|
$importLine = "`t_ `"github.com/banbox/banstagy/code`""
|
|
if ($mainText -notmatch [regex]::Escape('github.com/banbox/banstagy/code')) {
|
|
$mainText = [regex]::Replace($mainText, 'import \(\r?\n', "import (`r`n$importLine`r`n", 1)
|
|
Set-Content -LiteralPath $mainPath -Value $mainText -Encoding UTF8
|
|
}
|
|
$formatLog = Join-Path $script:LogRoot 'gofmt-integrate.log'
|
|
$format = Invoke-LoggedCommand -FilePath 'gofmt' -Arguments @('-w', $registryPath, $mainPath) -LogPath $formatLog
|
|
if ($format.ExitCode -ne 0) { throw "gofmt failed; see $formatLog" }
|
|
Set-Completed 'integrate'
|
|
Write-Host "Generated registry and integrated blank import in main.go"
|
|
}
|
|
|
|
function Invoke-Build {
|
|
$botFull = [IO.Path]::GetFullPath($BotPath)
|
|
$result = Invoke-LoggedCommand -FilePath 'go' -Arguments @('build', '-o', $botFull, '.') -LogPath (Join-Path $script:LogRoot 'build.log')
|
|
if ($result.ExitCode -ne 0 -or -not (Test-Path -LiteralPath $botFull)) {
|
|
throw "Build failed; see $($result.LogPath)"
|
|
}
|
|
Set-Completed 'build'
|
|
Write-Host "Build succeeded: $botFull"
|
|
}
|
|
|
|
function Remove-YamlTopLevelBlock {
|
|
param([string[]]$Lines, [Parameter(Mandatory)][string]$Name)
|
|
$result = New-Object Collections.Generic.List[string]
|
|
$skipping = $false
|
|
foreach ($line in $Lines) {
|
|
if ($line -match ('^' + [regex]::Escape($Name) + '\s*:')) {
|
|
$skipping = $true
|
|
continue
|
|
}
|
|
if ($skipping -and $line -match '^\S') { $skipping = $false }
|
|
if (-not $skipping) { $result.Add($line) }
|
|
}
|
|
return @($result)
|
|
}
|
|
|
|
function Set-YamlTopLevelScalar {
|
|
param([string[]]$Lines, [Parameter(Mandatory)][string]$Name, [Parameter(Mandatory)][string]$Value)
|
|
$filtered = @(Remove-YamlTopLevelBlock -Lines $Lines -Name $Name)
|
|
return @($filtered) + @("${Name}: $Value")
|
|
}
|
|
|
|
function Get-LegacyUniverseFromLog {
|
|
param([Parameter(Mandatory)]$Report)
|
|
$spec = $Report.dynamicUniverse
|
|
$logPath = Join-Path (Get-ReportRoot $Report) ([string]$spec.sourceLog)
|
|
$matches = New-Object Collections.Generic.List[object]
|
|
foreach ($line in Get-Content -LiteralPath $logPath) {
|
|
if ($line -match 'USDT\((?<count>\d+)\):\s+(?<bases>[A-Z0-9]+(?:\s+[A-Z0-9]+)*)\s*$') {
|
|
$bases = @($Matches.bases -split '\s+' | Where-Object { $_ })
|
|
if ($bases.Count -eq [int]$Matches.count) {
|
|
$matches.Add([pscustomobject]@{ Count = $bases.Count; Bases = $bases })
|
|
}
|
|
}
|
|
}
|
|
if ($spec.psobject.Properties['expectedWarmupPairs']) {
|
|
$filtered = @($matches | Where-Object Count -eq [int]$spec.expectedWarmupPairs)
|
|
$matches = New-Object Collections.Generic.List[object]
|
|
foreach ($match in $filtered) { $matches.Add($match) }
|
|
}
|
|
$selected = @($matches | Sort-Object Count -Descending | Select-Object -First 1)
|
|
if ($selected.Count -gt 0) {
|
|
return @($selected[0].Bases | ForEach-Object { "$_/USDT:USDT" })
|
|
}
|
|
$configPath = Join-Path (Get-ReportRoot $Report) 'config.yml'
|
|
foreach ($line in Get-Content -LiteralPath $configPath) {
|
|
if ($line -match '^pairs\s*:\s*\[(?<pairs>[^]]+)\]') {
|
|
$pairs = @($Matches.pairs -split ',' | ForEach-Object { $_.Trim().Trim("'", '"') } | Where-Object { $_ })
|
|
if ($pairs.Count -gt 0) {
|
|
return @($pairs | ForEach-Object { if ($_ -match '/') { $_ } else { "$_/USDT:USDT" } })
|
|
}
|
|
}
|
|
}
|
|
throw "Could not recover the legacy universe from $logPath or top-level pairs in $configPath"
|
|
}
|
|
|
|
function ConvertTo-SanitizedConfig {
|
|
param([Parameter(Mandatory)]$Report, [switch]$PinLegacyUniverse)
|
|
$sourcePath = Join-Path (Get-ReportRoot $Report) 'config.yml'
|
|
$lines = @(Get-Content -LiteralPath $sourcePath)
|
|
foreach ($block in @('api_server', 'rpc_channels', 'webhook', 'mail', 'llm_models')) {
|
|
$lines = @(Remove-YamlTopLevelBlock -Lines $lines -Name $block)
|
|
}
|
|
$sensitive = '(?i)^(?<indent>\s*)(?<key>api_key|api_secret|secret|token|password|passwd|pwd|jwt_secret_key|corp_secret|private_key|url)\s*:'
|
|
$lines = @($lines | ForEach-Object {
|
|
if ($_ -match $sensitive) { "$($Matches.indent)$($Matches.key): `"`"" } else { $_ }
|
|
})
|
|
foreach ($mapping in $Report.policyMap.psobject.Properties) {
|
|
$old = [regex]::Escape($mapping.Name)
|
|
$replacement = [string]$mapping.Value
|
|
$pattern = "^(?<prefix>\s*(?:-\s*)?name:\s*)$old(?<suffix>\s*(?:#.*)?)$"
|
|
$lines = @($lines | ForEach-Object {
|
|
if ($_ -match $pattern) { "$($Matches.prefix)$replacement$($Matches.suffix)" } else { $_ }
|
|
})
|
|
}
|
|
$detail = Get-Content -LiteralPath (Join-Path (Get-ReportRoot $Report) 'detail.json') -Raw | ConvertFrom-Json
|
|
foreach ($field in @('timerange', 'time_start', 'time_end')) {
|
|
$lines = @(Remove-YamlTopLevelBlock -Lines $lines -Name $field)
|
|
}
|
|
$lines += @("time_start: '$([int64]$detail.startMS)'", "time_end: '$([int64]$detail.endMS)'")
|
|
if ($PinLegacyUniverse -and $Report.psobject.Properties['dynamicUniverse']) {
|
|
$pairs = @(Get-LegacyUniverseFromLog -Report $Report)
|
|
$lines = @(Remove-YamlTopLevelBlock -Lines $lines -Name 'pairs')
|
|
$lines = @(Remove-YamlTopLevelBlock -Lines $lines -Name 'pairlists')
|
|
$lines = @($lines | ForEach-Object {
|
|
if ($_ -match '^(?<indent>\s*)use_latest\s*:') { "$($Matches.indent)use_latest: false" } else { $_ }
|
|
})
|
|
$quotedPairs = @($pairs | ForEach-Object { "'" + $_.Replace("'", "''") + "'" }) -join ', '
|
|
$lines += "pairs: [$quotedPairs]"
|
|
}
|
|
$lines += @('', 'api_server:', ' enable: false', 'rpc_channels: {}', 'webhook: {}', 'llm_models: {}')
|
|
$configRoot = [IO.Path]::GetFullPath((Join-Path $script:RepoRoot $script:Manifest.sanitizedConfigRoot))
|
|
New-Item -ItemType Directory -Force -Path $configRoot | Out-Null
|
|
$destination = Join-Path $configRoot "$($Report.id).yml"
|
|
Set-Content -LiteralPath $destination -Value $lines -Encoding UTF8
|
|
return $destination
|
|
}
|
|
|
|
function Get-NestedYamlScalar {
|
|
param([Parameter(Mandatory)][string]$Path, [Parameter(Mandatory)][string]$Section, [Parameter(Mandatory)][string]$Key)
|
|
$inside = $false
|
|
foreach ($line in Get-Content -LiteralPath $Path) {
|
|
if ($line -match ('^' + [regex]::Escape($Section) + '\s*:')) { $inside = $true; continue }
|
|
if ($inside -and $line -match '^\S') { break }
|
|
if ($inside -and $line -match ('^\s+' + [regex]::Escape($Key) + '\s*:\s*(?<value>.*?)\s*$')) {
|
|
return $Matches.value.Trim('''', '"')
|
|
}
|
|
}
|
|
return ''
|
|
}
|
|
|
|
function Resolve-DatabaseUrl {
|
|
if ($DatabaseUrl) { return $DatabaseUrl }
|
|
if ($DatabaseOverlay) {
|
|
$overlayFull = [IO.Path]::GetFullPath($DatabaseOverlay)
|
|
if (-not (Test-Path -LiteralPath $overlayFull -PathType Leaf)) { throw "Database overlay not found: $overlayFull" }
|
|
$value = Get-NestedYamlScalar -Path $overlayFull -Section 'database' -Key 'url'
|
|
if ($value) { return $value }
|
|
}
|
|
throw 'Database connection is required. Pass a database-only -DatabaseOverlay, -DatabaseUrl, or set BANBOT_DATABASE_URL.'
|
|
}
|
|
|
|
function Get-DatabaseEndpoint {
|
|
param([Parameter(Mandatory)][string]$Url)
|
|
try {
|
|
$uri = [Uri]$Url
|
|
if ($uri.Host) {
|
|
$port = $uri.Port
|
|
if ($port -le 0) { $port = if ($DatabaseType -eq 'questdb') { 8812 } else { 5432 } }
|
|
return [pscustomobject]@{ Host = $uri.Host; Port = $port }
|
|
}
|
|
} catch { }
|
|
$hostMatch = [regex]::Match($Url, '(?:^|\s)host=(?<host>[^\s]+)', 'IgnoreCase')
|
|
$portMatch = [regex]::Match($Url, '(?:^|\s)port=(?<port>\d+)', 'IgnoreCase')
|
|
if ($hostMatch.Success) {
|
|
$port = if ($portMatch.Success) { [int]$portMatch.Groups['port'].Value } elseif ($DatabaseType -eq 'questdb') { 8812 } else { 5432 }
|
|
return [pscustomobject]@{ Host = $hostMatch.Groups['host'].Value; Port = $port }
|
|
}
|
|
throw 'Cannot determine database host/port from the supplied URL.'
|
|
}
|
|
|
|
function Assert-DatabaseAvailable {
|
|
param([Parameter(Mandatory)][string]$Url)
|
|
$endpoint = Get-DatabaseEndpoint $Url
|
|
$client = New-Object Net.Sockets.TcpClient
|
|
try {
|
|
$task = $client.ConnectAsync($endpoint.Host, $endpoint.Port)
|
|
$available = $task.Wait(3000) -and $client.Connected
|
|
} catch {
|
|
$available = $false
|
|
} finally {
|
|
$client.Dispose()
|
|
}
|
|
if ($available) {
|
|
Write-Host "Database preflight passed at $($endpoint.Host):$($endpoint.Port)"
|
|
} else {
|
|
Write-Warning "Database is not currently reachable at $($endpoint.Host):$($endpoint.Port); banbot may auto-start the configured QuestDB service. The backtest exit and artifacts remain the hard check."
|
|
}
|
|
return $available
|
|
}
|
|
|
|
function New-DatabaseRuntimeOverlay {
|
|
param([Parameter(Mandatory)][string]$Url)
|
|
if ($DatabaseOverlay) {
|
|
$overlayFull = [IO.Path]::GetFullPath($DatabaseOverlay)
|
|
$topKeys = @(Get-Content -LiteralPath $overlayFull | Where-Object { $_ -match '^\S[^#]*:' } | ForEach-Object { ($_ -split ':', 2)[0].Trim() } | Sort-Object -Unique)
|
|
if ($topKeys.Count -ne 1 -or $topKeys[0] -ne 'database') {
|
|
throw '-DatabaseOverlay must contain only the top-level database section; full local configs are rejected to prevent credential merging.'
|
|
}
|
|
return [pscustomobject]@{ Path = $overlayFull; Temporary = $false }
|
|
}
|
|
$quotedUrl = "'" + $Url.Replace("'", "''") + "'"
|
|
$lines = @('database:', " url: $quotedUrl")
|
|
if ($DatabaseType) { $lines += " db_type: $DatabaseType" }
|
|
$runtimePath = Join-Path ([IO.Path]::GetTempPath()) ("banstagy-$([Guid]::NewGuid().ToString('N')).yml")
|
|
Set-Content -LiteralPath $runtimePath -Value $lines -Encoding UTF8
|
|
return [pscustomobject]@{ Path = $runtimePath; Temporary = $true }
|
|
}
|
|
|
|
function Set-RecentWindow {
|
|
param([Parameter(Mandatory)][string]$ConfigPath, [Parameter(Mandatory)][string]$Destination)
|
|
$lines = @(Get-Content -LiteralPath $ConfigPath)
|
|
foreach ($field in @('timerange', 'time_start', 'time_end')) {
|
|
$lines = @(Remove-YamlTopLevelBlock -Lines $lines -Name $field)
|
|
}
|
|
$today = $script:RecentToday
|
|
$start = $today.AddYears(-2)
|
|
$lines += @("time_start: '$($start.ToString('yyyyMMdd'))'", "time_end: '$($today.ToString('yyyyMMdd'))'")
|
|
Set-Content -LiteralPath $Destination -Value $lines -Encoding UTF8
|
|
return [pscustomobject]@{ Start = $start.ToString('yyyyMMdd'); End = $today.ToString('yyyyMMdd') }
|
|
}
|
|
|
|
function Get-RequiredProperty {
|
|
param([Parameter(Mandatory)]$Object, [Parameter(Mandatory)][string]$Name, [Parameter(Mandatory)][string]$Path)
|
|
$property = $Object.psobject.Properties[$Name]
|
|
if ($null -eq $property) { throw "Required detail field missing: $Path.$Name" }
|
|
return $property.Value
|
|
}
|
|
|
|
function Get-ComparableGroupTitle {
|
|
param(
|
|
[Parameter(Mandatory)]$Report,
|
|
[Parameter(Mandatory)][string]$GroupName,
|
|
[Parameter(Mandatory)][string]$Title,
|
|
[switch]$Expected
|
|
)
|
|
if (-not $Expected -or $GroupName -notin @('enterGrps', 'exitGrps')) { return $Title }
|
|
foreach ($mapping in $Report.policyMap.psobject.Properties) {
|
|
$legacy = [string]$mapping.Name
|
|
if ($Title -eq $legacy) { return [string]$mapping.Value }
|
|
if ($Title.StartsWith("${legacy}:", [StringComparison]::Ordinal)) {
|
|
return ([string]$mapping.Value) + $Title.Substring($legacy.Length)
|
|
}
|
|
}
|
|
return $Title
|
|
}
|
|
|
|
function Compare-Detail {
|
|
param(
|
|
[Parameter(Mandatory)]$Report,
|
|
[Parameter(Mandatory)][string]$ExpectedPath,
|
|
[Parameter(Mandatory)][string]$ActualPath
|
|
)
|
|
$expected = Get-Content -LiteralPath $ExpectedPath -Raw | ConvertFrom-Json
|
|
$actual = Get-Content -LiteralPath $ActualPath -Raw | ConvertFrom-Json
|
|
$diffs = New-Object Collections.Generic.List[object]
|
|
$universeDiverged = $false
|
|
|
|
foreach ($field in @($script:Manifest.comparison.integerFields)) {
|
|
$left = Get-RequiredProperty -Object $expected -Name $field -Path '$'
|
|
$right = Get-RequiredProperty -Object $actual -Name $field -Path '$'
|
|
if ([decimal]$left -ne [decimal]$right) {
|
|
$diffs.Add([pscustomobject]@{ path = "`$.$field"; kind = 'integer'; expected = $left; actual = $right; delta = ([decimal]$right - [decimal]$left) })
|
|
}
|
|
}
|
|
foreach ($field in @($script:Manifest.comparison.floatFields)) {
|
|
$left = Get-RequiredProperty -Object $expected -Name $field -Path '$'
|
|
$right = Get-RequiredProperty -Object $actual -Name $field -Path '$'
|
|
$delta = [Math]::Abs([double]$right - [double]$left)
|
|
if ($delta -gt $FloatTolerance) {
|
|
$diffs.Add([pscustomobject]@{ path = "`$.$field"; kind = 'float'; expected = $left; actual = $right; delta = $delta })
|
|
}
|
|
}
|
|
foreach ($groupName in @($script:Manifest.comparison.groups)) {
|
|
$leftProperty = $expected.psobject.Properties[$groupName]
|
|
$rightProperty = $actual.psobject.Properties[$groupName]
|
|
if ($null -eq $leftProperty -and $null -eq $rightProperty) { continue }
|
|
if ($null -eq $leftProperty -or $null -eq $rightProperty) {
|
|
$diffs.Add([pscustomobject]@{ path = "`$.$groupName"; kind = 'schema'; expected = ($null -ne $leftProperty); actual = ($null -ne $rightProperty); delta = $null })
|
|
continue
|
|
}
|
|
$leftGroup = @($leftProperty.Value)
|
|
$rightGroup = @($rightProperty.Value)
|
|
$key = [string]$script:Manifest.comparison.groupKey
|
|
$leftMap = @{}; $rightMap = @{}
|
|
foreach ($row in $leftGroup) {
|
|
$title = [string](Get-RequiredProperty $row $key "`$.$groupName")
|
|
$title = Get-ComparableGroupTitle -Report $Report -GroupName $groupName -Title $title -Expected
|
|
$leftMap[$title] = $row
|
|
}
|
|
foreach ($row in $rightGroup) { $rightMap[[string](Get-RequiredProperty $row $key "`$.$groupName")] = $row }
|
|
foreach ($title in @($leftMap.Keys + $rightMap.Keys | Sort-Object -Unique)) {
|
|
if (-not $leftMap.ContainsKey($title) -or -not $rightMap.ContainsKey($title)) {
|
|
$diffs.Add([pscustomobject]@{ path = "`$.$groupName[$title]"; kind = 'group'; expected = $leftMap.ContainsKey($title); actual = $rightMap.ContainsKey($title); delta = $null })
|
|
if ($groupName -eq 'pairGrps') { $universeDiverged = $true }
|
|
continue
|
|
}
|
|
foreach ($field in @($script:Manifest.comparison.groupIntegerFields)) {
|
|
$left = Get-RequiredProperty $leftMap[$title] $field "`$.$groupName[$title]"
|
|
$right = Get-RequiredProperty $rightMap[$title] $field "`$.$groupName[$title]"
|
|
if ([decimal]$left -ne [decimal]$right) {
|
|
$diffs.Add([pscustomobject]@{ path = "`$.$groupName[$title].$field"; kind = 'integer'; expected = $left; actual = $right; delta = ([decimal]$right - [decimal]$left) })
|
|
}
|
|
}
|
|
foreach ($field in @($script:Manifest.comparison.groupFloatFields)) {
|
|
$left = Get-RequiredProperty $leftMap[$title] $field "`$.$groupName[$title]"
|
|
$right = Get-RequiredProperty $rightMap[$title] $field "`$.$groupName[$title]"
|
|
$delta = [Math]::Abs([double]$right - [double]$left)
|
|
if ($delta -gt $FloatTolerance) {
|
|
$diffs.Add([pscustomobject]@{ path = "`$.$groupName[$title].$field"; kind = 'float'; expected = $left; actual = $right; delta = $delta })
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return [pscustomobject]@{ matched = ($diffs.Count -eq 0); differenceCount = $diffs.Count; universeDiverged = $universeDiverged; differences = @($diffs | ForEach-Object { $_ }) }
|
|
}
|
|
|
|
function Get-RedactedLogTail {
|
|
param([Parameter(Mandatory)][string]$Path)
|
|
if (-not (Test-Path -LiteralPath $Path)) { return '' }
|
|
$text = (@(Get-Content -LiteralPath $Path -Tail 250) -join "`n")
|
|
$text = $text -replace '(?i)(postgres(?:ql)?://[^:/\s]+:)[^@\s]+@', '$1<redacted>@'
|
|
$text = $text -replace '(?i)(api[_-]?key|api[_-]?secret|token|password|pwd|secret)\s*[:=]\s*\S+', '$1=<redacted>'
|
|
return $text
|
|
}
|
|
|
|
function Clear-DatabaseUrlInConfig {
|
|
param([Parameter(Mandatory)][string]$Path)
|
|
$lines = @(Get-Content -LiteralPath $Path | ForEach-Object {
|
|
if ($_ -match '^(?<indent>\s*)url\s*:') { "$($Matches.indent)url: `"`"" } else { $_ }
|
|
})
|
|
Set-Content -LiteralPath $Path -Value $lines -Encoding UTF8
|
|
}
|
|
|
|
function Assert-YamlParseable {
|
|
param([Parameter(Mandatory)][string]$Path, [Parameter(Mandatory)][string]$LogPath)
|
|
$helper = Join-Path $script:WorkRoot 'yamlcheck.go'
|
|
if (-not (Test-Path -LiteralPath $helper)) {
|
|
@'
|
|
package main
|
|
|
|
import (
|
|
"os"
|
|
"gopkg.in/yaml.v3"
|
|
)
|
|
|
|
func main() {
|
|
data, err := os.ReadFile(os.Args[1])
|
|
if err != nil { panic(err) }
|
|
var value any
|
|
if err = yaml.Unmarshal(data, &value); err != nil { panic(err) }
|
|
}
|
|
'@ | Set-Content -LiteralPath $helper -Encoding UTF8
|
|
}
|
|
$result = Invoke-LoggedCommand -FilePath 'go' -Arguments @('run', $helper, $Path) -LogPath $LogPath -TimeoutSeconds 120
|
|
if ($result.ExitCode -ne 0) { throw "Generated config is not parseable YAML; see $LogPath" }
|
|
}
|
|
|
|
function Assert-BacktestArtifacts {
|
|
param([Parameter(Mandatory)][string]$OutputDir, [Parameter(Mandatory)][DateTime]$StartedUtc)
|
|
$paths = @{
|
|
detail = Join-Path $OutputDir 'detail.json'
|
|
config = Join-Path $OutputDir 'config.yml'
|
|
outLog = Join-Path $OutputDir 'out.log'
|
|
}
|
|
foreach ($entry in $paths.GetEnumerator()) {
|
|
if (-not (Test-Path -LiteralPath $entry.Value -PathType Leaf)) { throw "Backtest did not produce $($entry.Key): $($entry.Value)" }
|
|
if ((Get-Item -LiteralPath $entry.Value).LastWriteTimeUtc -lt $StartedUtc.AddSeconds(-2)) { throw "Backtest artifact is stale: $($entry.Value)" }
|
|
}
|
|
Clear-DatabaseUrlInConfig -Path $paths.config
|
|
Assert-YamlParseable -Path $paths.config -LogPath (Join-Path $OutputDir 'yaml-parse.log')
|
|
try { $detail = Get-Content -LiteralPath $paths.detail -Raw | ConvertFrom-Json }
|
|
catch { throw "detail.json is not parseable: $($paths.detail): $($_.Exception.Message)" }
|
|
$expectedOut = [IO.Path]::GetFullPath($OutputDir).TrimEnd('\', '/')
|
|
$actualOut = [IO.Path]::GetFullPath([string](Get-RequiredProperty $detail 'outDir' '$')).TrimEnd('\', '/')
|
|
if (-not $actualOut.Equals($expectedOut, [StringComparison]::OrdinalIgnoreCase)) {
|
|
throw "detail.outDir mismatch: expected $expectedOut, got $actualOut"
|
|
}
|
|
$numericFields = @($script:Manifest.comparison.integerFields) + @($script:Manifest.comparison.floatFields)
|
|
foreach ($field in $numericFields) {
|
|
$value = [double](Get-RequiredProperty $detail $field '$')
|
|
if ([double]::IsNaN($value) -or [double]::IsInfinity($value)) { throw "Non-finite core metric in detail.json: $field" }
|
|
}
|
|
$calcDiff = $null
|
|
if ($detail.psobject.Properties['calcDiff']) {
|
|
$calcDiff = [double]$detail.calcDiff
|
|
if ([Math]::Abs($calcDiff) -gt $FloatTolerance) { Write-Warning "Backtest reports calcDiff=$calcDiff in $OutputDir" }
|
|
}
|
|
return [pscustomobject]@{ detail = $paths.detail; config = $paths.config; outLog = $paths.outLog; calcDiff = $calcDiff }
|
|
}
|
|
|
|
function Invoke-Backtest {
|
|
param([Parameter(Mandatory)]$Report, [Parameter(Mandatory)][string]$SanitizedConfig, [Parameter(Mandatory)][string]$OutputDir, [Parameter(Mandatory)][string]$Label, [Parameter(Mandatory)][string]$DbUrl, [switch]$RequireOrders, [switch]$NetworkOff)
|
|
$runRoot = [IO.Path]::GetFullPath((Join-Path $script:RepoRoot $script:Manifest.runRoot))
|
|
Assert-DescendantPath -Root $runRoot -Path $OutputDir | Out-Null
|
|
New-Item -ItemType Directory -Force -Path $OutputDir | Out-Null
|
|
$uniqueName = "$Label-$([DateTime]::UtcNow.ToString('yyyyMMddTHHmmssfffZ'))-$([Guid]::NewGuid().ToString('N').Substring(0, 8))"
|
|
$OutputDir = Join-Path $OutputDir $uniqueName
|
|
Assert-DescendantPath -Root $runRoot -Path $OutputDir | Out-Null
|
|
if (Test-Path -LiteralPath $OutputDir) { throw "Unique output already exists: $OutputDir" }
|
|
New-Item -ItemType Directory -Path $OutputDir | Out-Null
|
|
$runtimeOverlay = New-DatabaseRuntimeOverlay -Url $DbUrl
|
|
$startedUtc = [DateTime]::UtcNow
|
|
try {
|
|
$captureLog = Join-Path $OutputDir 'process.log'
|
|
$args = @('backtest', '-datadir', [string]$script:Manifest.dataDir, '-no-default', '-config', $SanitizedConfig, '-config', $runtimeOverlay.Path, '-out', $OutputDir)
|
|
if ($NetworkOff) { $args += '-net-off' }
|
|
$result = Invoke-LoggedCommand -FilePath ([IO.Path]::GetFullPath($BotPath)) -Arguments $args -LogPath $captureLog -TimeoutSeconds ($BacktestTimeoutMinutes * 60)
|
|
if ($result.ExitCode -ne 0) { throw "Backtest $Label failed for $($Report.id); see $captureLog and $(Join-Path $OutputDir 'out.log')" }
|
|
$artifacts = Assert-BacktestArtifacts -OutputDir $OutputDir -StartedUtc $startedUtc
|
|
if ($RequireOrders) {
|
|
foreach ($name in @('orders.csv', 'orders.gob')) {
|
|
if (-not (Test-Path -LiteralPath (Join-Path $OutputDir $name) -PathType Leaf)) { throw "Recent backtest did not produce required $name in $OutputDir" }
|
|
}
|
|
}
|
|
return [pscustomobject]@{ output = $OutputDir; detail = $artifacts.detail; botLog = $artifacts.outLog; processLog = $captureLog; startedUtc = $startedUtc.ToString('o'); exitCode = $result.ExitCode; calcDiff = $artifacts.calcDiff }
|
|
} finally {
|
|
$emittedConfig = Join-Path $OutputDir 'config.yml'
|
|
if (Test-Path -LiteralPath $emittedConfig -PathType Leaf) { Clear-DatabaseUrlInConfig -Path $emittedConfig }
|
|
if ($runtimeOverlay.Temporary -and (Test-Path -LiteralPath $runtimeOverlay.Path)) { Remove-Item -LiteralPath $runtimeOverlay.Path -Force }
|
|
}
|
|
}
|
|
|
|
function Invoke-CodexRepair {
|
|
param([Parameter(Mandatory)]$Report, [Parameter(Mandatory)]$Comparison, [Parameter(Mandatory)][string]$BotLog)
|
|
$scratch = Join-Path $script:WorkRoot "repair-$($Report.id)"
|
|
Copy-MigrationInputs -Report $Report -Scratch $scratch
|
|
$generated = Join-Path (Join-Path $script:RepoRoot $script:Manifest.generatedRoot) $Report.id
|
|
Copy-Item -LiteralPath $generated -Destination (Join-Path $scratch 'current') -Recurse
|
|
$diffJson = $Comparison | ConvertTo-Json -Depth 20
|
|
$logTail = Get-RedactedLogTail -Path $BotLog
|
|
$blocks = New-Object Collections.Generic.List[string]
|
|
$blocks.Add((Get-InlineFileBlock -Label 'banbot_cn.md' -Path (Join-Path $scratch 'input\banbot_cn.md')))
|
|
foreach ($relative in @($Report.sourceFiles)) {
|
|
$blocks.Add((Get-InlineFileBlock -Label "legacy/$($relative.Replace('\', '/'))" -Path (Join-Path (Join-Path $scratch 'input\sources') $relative)))
|
|
}
|
|
foreach ($file in @(Get-ChildItem -LiteralPath (Join-Path $scratch 'current') -File -Filter '*.go' -Recurse)) {
|
|
$relative = Get-RelativePath -Base (Join-Path $scratch 'current') -Path $file.FullName
|
|
$blocks.Add((Get-InlineFileBlock -Label "current/$($relative.Replace('\', '/'))" -Path $file.FullName))
|
|
}
|
|
$inlineInputs = $blocks -join "`n`n"
|
|
$basePrompt = @"
|
|
Do not call tools or inspect the filesystem. All allowed inputs are included below.
|
|
Return only JSON matching the supplied schema, containing the full replacement Go package. Each path must be relative and end in .go.
|
|
Repair the migrated banbot strategy so its backtest reproduces the legacy metrics.
|
|
Preserve group $($Report.generatedGroup) and these mappings:
|
|
$(Get-PolicyInstructions $Report)
|
|
|
|
Diagnose OnData main-vs-info routing, warmup, timeframe, order timing, and state behavior. Make the smallest behavior-preserving fix. Do not weaken comparisons or alter metrics. Do not add dependencies. Do not include markdown fences or explanations.
|
|
|
|
METRIC DIFFERENCES:
|
|
$diffJson
|
|
|
|
REDACTED LOG TAIL:
|
|
$logTail
|
|
|
|
$inlineInputs
|
|
"@
|
|
$schemaPath = Join-Path $scratch 'files-schema.json'
|
|
$replacement = Join-Path $scratch 'replacement'
|
|
Write-CodexFilesSchema -Path $schemaPath
|
|
$reasoningConfig = 'model_reasoning_effort="' + $CodexReasoning + '"'
|
|
$validationFeedback = ''
|
|
$validated = $false
|
|
for ($validationAttempt = 0; $validationAttempt -le $MaxCodexValidationAttempts; $validationAttempt++) {
|
|
$prompt = $basePrompt
|
|
if ($validationFeedback) {
|
|
$prompt += @"
|
|
|
|
PREVIOUS RESPONSE VALIDATION ERROR:
|
|
$validationFeedback
|
|
|
|
Return a corrected full replacement package. Deprecated OnBar and OnInfoBar callbacks must not appear as struct fields.
|
|
"@
|
|
}
|
|
$attemptStamp = [DateTime]::UtcNow.ToString('yyyyMMddHHmmssfff')
|
|
$logPath = Join-Path $script:LogRoot "codex-repair-$($Report.id)-$attemptStamp-attempt$validationAttempt.log"
|
|
$responsePath = Join-Path $scratch "files-response-$validationAttempt.json"
|
|
try {
|
|
$result = Invoke-LoggedCommand -FilePath 'codex' -Arguments @('exec', '--disable', 'hooks', '--disable', 'multi_agent', '-c', $reasoningConfig, '--sandbox', 'read-only', '--ephemeral', '--skip-git-repo-check', '--output-schema', $schemaPath, '--output-last-message', $responsePath, '-C', $scratch, '-') -LogPath $logPath -InputText $prompt -WorkingDirectory $scratch
|
|
if ($result.ExitCode -ne 0) { throw "Codex repair command failed; see $logPath" }
|
|
if (Test-Path -LiteralPath $replacement) { Remove-Item -LiteralPath $replacement -Recurse -Force }
|
|
Write-CodexResponseFiles -ResponsePath $responsePath -Destination $replacement
|
|
$formatArgs = @('-w') + @(Get-ChildItem -LiteralPath $replacement -File -Filter '*.go' -Recurse | Select-Object -ExpandProperty FullName)
|
|
$formatLog = Join-Path $script:LogRoot "gofmt-repair-$($Report.id)-attempt$validationAttempt.log"
|
|
$format = Invoke-LoggedCommand -FilePath 'gofmt' -Arguments $formatArgs -LogPath $formatLog
|
|
if ($format.ExitCode -ne 0) { throw "gofmt repair failed; see $formatLog" }
|
|
Assert-GeneratedPackage -Report $Report -Directory $replacement
|
|
$validated = $true
|
|
break
|
|
} catch {
|
|
$validationFeedback = $_.Exception.Message
|
|
if ($validationAttempt -ge $MaxCodexValidationAttempts) {
|
|
throw "Codex repair validation failed for $($Report.id) after $($validationAttempt + 1) attempts: $validationFeedback"
|
|
}
|
|
Write-Warning "Retrying Codex repair for $($Report.id): $validationFeedback"
|
|
}
|
|
}
|
|
if (-not $validated) { throw "Codex repair did not produce a validated package for $($Report.id)" }
|
|
Assert-DescendantPath -Root (Join-Path $script:RepoRoot $script:Manifest.generatedRoot) -Path $generated | Out-Null
|
|
Remove-Item -LiteralPath $generated -Recurse -Force
|
|
Copy-Item -LiteralPath $replacement -Destination $generated -Recurse
|
|
Invoke-Build
|
|
}
|
|
|
|
function Read-Summary {
|
|
if (Test-Path -LiteralPath $script:SummaryPath) {
|
|
return Get-Content -LiteralPath $script:SummaryPath -Raw | ConvertFrom-Json
|
|
}
|
|
return [pscustomobject]@{
|
|
schemaVersion = 1
|
|
generatedUtc = [DateTime]::UtcNow.ToString('o')
|
|
floatTolerance = $FloatTolerance
|
|
recentWindow = $null
|
|
reports = @()
|
|
}
|
|
}
|
|
|
|
function Save-ReportSummary {
|
|
param([Parameter(Mandatory)]$ReportResult, $RecentWindow = $null)
|
|
$summary = Read-Summary
|
|
$summary.generatedUtc = [DateTime]::UtcNow.ToString('o')
|
|
$summary.floatTolerance = $FloatTolerance
|
|
if ($null -ne $RecentWindow) { $summary.recentWindow = $RecentWindow }
|
|
$summary.reports = @(@($summary.reports | Where-Object id -ne $ReportResult.id) + $ReportResult | Sort-Object id)
|
|
Write-JsonAtomic -Value $summary -Path $script:SummaryPath
|
|
}
|
|
|
|
function Invoke-Replay {
|
|
$dbUrl = Resolve-DatabaseUrl
|
|
$null = Assert-DatabaseAvailable -Url $dbUrl
|
|
$failures = New-Object Collections.Generic.List[string]
|
|
foreach ($report in $script:Representatives) {
|
|
if (-not $Force -and (Test-Completed "replay:$($report.id)")) {
|
|
Write-Host "Resume: baseline replay already matched for $($report.id)"
|
|
continue
|
|
}
|
|
$sanitized = ConvertTo-SanitizedConfig -Report $report -PinLegacyUniverse
|
|
$attempts = New-Object Collections.Generic.List[object]
|
|
$comparison = $null
|
|
$exactReplayBlocker = $null
|
|
for ($attempt = 0; $attempt -le $MaxRepairAttempts; $attempt++) {
|
|
$output = Join-Path (Join-Path (Join-Path $script:RepoRoot $script:Manifest.runRoot) $report.id) 'baseline'
|
|
$run = Invoke-Backtest -Report $report -SanitizedConfig $sanitized -OutputDir $output -Label "baseline-$attempt" -DbUrl $dbUrl -NetworkOff
|
|
$comparison = Compare-Detail -Report $report -ExpectedPath (Join-Path (Get-ReportRoot $report) 'detail.json') -ActualPath $run.detail
|
|
$attempts.Add([pscustomobject]@{ attempt = $attempt; startedUtc = $run.startedUtc; exitCode = $run.exitCode; matched = $comparison.matched; differenceCount = $comparison.differenceCount; calcDiffWarning = $run.calcDiff; output = $run.output; differences = @($comparison.differences) })
|
|
if ($comparison.matched) { break }
|
|
$eventCountDiffs = @($comparison.differences | Where-Object { $_.path -in @('$.barNum', '$.timeNum') })
|
|
if ($eventCountDiffs.Count -gt 0) {
|
|
$exactReplayBlocker = "Archived event counts differ from the current banbot/data environment ($(@($eventCountDiffs | ForEach-Object { $_.path + ': ' + $_.expected + ' -> ' + $_.actual }) -join ', ')); strategy repair is disabled."
|
|
break
|
|
}
|
|
if ($comparison.universeDiverged -and $Report.psobject.Properties['dynamicUniverse']) {
|
|
$exactReplayBlocker = "The title-keyed pair set diverged for dynamic $($report.dynamicUniverse.source) after pinning the universe from $($report.dynamicUniverse.sourceLog); strategy repair is not appropriate."
|
|
break
|
|
}
|
|
if ($attempt -lt $MaxRepairAttempts) { Invoke-CodexRepair -Report $report -Comparison $comparison -BotLog $run.botLog }
|
|
}
|
|
$baselineDetail = Get-Content -LiteralPath (Join-Path (Get-ReportRoot $report) 'detail.json') -Raw | ConvertFrom-Json
|
|
$universePin = if ($Report.psobject.Properties['dynamicUniverse']) { $Report.dynamicUniverse } else { $null }
|
|
$result = [pscustomobject]@{
|
|
id = $report.id
|
|
baselineMatched = $comparison.matched
|
|
baselineWindowOverride = [pscustomobject]@{ startMS = [int64]$baselineDetail.startMS; endMS = [int64]$baselineDetail.endMS }
|
|
dynamicUniverse = $universePin
|
|
exactReplayBlocker = $exactReplayBlocker
|
|
baselineAttempts = @($attempts | ForEach-Object { $_ })
|
|
recent = $null
|
|
}
|
|
Save-ReportSummary -ReportResult $result
|
|
if (-not $comparison.matched) {
|
|
if ($exactReplayBlocker) {
|
|
Set-Completed "replay:$($report.id)"
|
|
Set-Completed "replay-blocked:$($report.id)"
|
|
Write-Warning "Baseline evaluated with an environment blocker for $($report.id): $exactReplayBlocker"
|
|
continue
|
|
}
|
|
$failures.Add($report.id)
|
|
Write-Warning "Baseline did not match for $($report.id); see $script:SummaryPath"
|
|
continue
|
|
}
|
|
Set-Completed "replay:$($report.id)"
|
|
Write-Host "Baseline matched: $($report.id)"
|
|
}
|
|
if ($failures.Count -gt 0) { throw "Baseline replay incomplete for: $($failures -join ', '). See $script:SummaryPath" }
|
|
}
|
|
|
|
function Remove-RecentOrderArtifacts {
|
|
param([Parameter(Mandatory)][string]$OutputDir)
|
|
$recentRoot = [IO.Path]::GetFullPath((Join-Path $script:RepoRoot $script:Manifest.runRoot))
|
|
Assert-NoReparsePath -Root $recentRoot -Path $OutputDir | Out-Null
|
|
foreach ($name in @('orders.csv', 'orders.gob')) {
|
|
$path = Join-Path $OutputDir $name
|
|
Assert-NoReparsePath -Root $OutputDir -Path $path | Out-Null
|
|
if (-not (Test-Path -LiteralPath $path -PathType Leaf)) { throw "Expected recent artifact before cleanup: $path" }
|
|
Remove-Item -LiteralPath $path -Force
|
|
if (Test-Path -LiteralPath $path) { throw "Failed to delete recent artifact: $path" }
|
|
}
|
|
}
|
|
|
|
function Invoke-Recent {
|
|
$dbUrl = Resolve-DatabaseUrl
|
|
$null = Assert-DatabaseAvailable -Url $dbUrl
|
|
foreach ($report in $script:Representatives) {
|
|
if (-not (Test-Completed "replay:$($report.id)")) { throw "Recent run requires a matched baseline for $($report.id)" }
|
|
if (-not $Force -and (Test-Completed "recent:$($report.id)")) {
|
|
Write-Host "Resume: recent replay already complete for $($report.id)"
|
|
continue
|
|
}
|
|
$sanitized = ConvertTo-SanitizedConfig -Report $report
|
|
$recentConfig = Join-Path ([IO.Path]::GetTempPath()) ("banstagy-recent-$([Guid]::NewGuid().ToString('N')).yml")
|
|
try {
|
|
$window = Set-RecentWindow -ConfigPath $sanitized -Destination $recentConfig
|
|
$output = Join-Path (Join-Path (Join-Path $script:RepoRoot $script:Manifest.runRoot) $report.id) 'recent'
|
|
$run = Invoke-Backtest -Report $report -SanitizedConfig $recentConfig -OutputDir $output -Label 'recent' -DbUrl $dbUrl -RequireOrders
|
|
$recentDetail = Get-Content -LiteralPath $run.detail -Raw | ConvertFrom-Json
|
|
$startUtc = [DateTime]::SpecifyKind($script:RecentToday.AddYears(-2), [DateTimeKind]::Utc)
|
|
$endUtc = [DateTime]::SpecifyKind($script:RecentToday, [DateTimeKind]::Utc)
|
|
$startMS = ([DateTimeOffset]$startUtc).ToUnixTimeMilliseconds()
|
|
$endMS = ([DateTimeOffset]$endUtc).ToUnixTimeMilliseconds()
|
|
if ([int64]$recentDetail.startMS -ne $startMS -or [int64]$recentDetail.endMS -ne $endMS) {
|
|
throw "Recent detail range does not exactly match UTC [$startMS,$endMS): actual $($recentDetail.startMS)-$($recentDetail.endMS)"
|
|
}
|
|
Remove-RecentOrderArtifacts -OutputDir $run.output
|
|
$summary = Read-Summary
|
|
$existing = @($summary.reports | Where-Object id -eq $report.id | Select-Object -First 1)
|
|
if ($existing.Count -eq 0) { throw "Missing baseline summary for $($report.id)" }
|
|
$existing[0].recent = [pscustomobject]@{ output = $run.output; detail = $run.detail; startedUtc = $run.startedUtc; exitCode = $run.exitCode; calcDiffWarning = $run.calcDiff; start = $window.Start; endExclusive = $window.End; ordersRemoved = $true }
|
|
Save-ReportSummary -ReportResult $existing[0] -RecentWindow ([pscustomobject]@{ start = $window.Start; endExclusive = $window.End; timezone = 'UTC' })
|
|
Set-Completed "recent:$($report.id)"
|
|
Write-Host "Recent two-year replay complete: $($report.id)"
|
|
} finally {
|
|
if (Test-Path -LiteralPath $recentConfig) { Remove-Item -LiteralPath $recentConfig -Force }
|
|
}
|
|
}
|
|
}
|
|
|
|
New-Item -ItemType Directory -Force -Path $script:WorkRoot, $script:LogRoot | Out-Null
|
|
$script:Representatives = @(Resolve-Representatives)
|
|
foreach ($report in $script:Representatives) { Assert-RepresentativeInputs $report }
|
|
|
|
switch ($Phase) {
|
|
'Inventory' { Invoke-Inventory }
|
|
'Migrate' { Invoke-Migrate }
|
|
'Integrate' { Invoke-Integrate }
|
|
'Build' { Invoke-Build }
|
|
'Replay' { Invoke-Replay }
|
|
'Recent' { Invoke-Recent }
|
|
'All' {
|
|
Invoke-Inventory
|
|
Invoke-Migrate
|
|
Invoke-Integrate
|
|
Invoke-Build
|
|
Invoke-Replay
|
|
Invoke-Recent
|
|
}
|
|
}
|