diff --git a/.scripts/build_with_toolchain.ps1 b/.scripts/build_with_toolchain.ps1 new file mode 100644 index 0000000..83d6759 --- /dev/null +++ b/.scripts/build_with_toolchain.ps1 @@ -0,0 +1,59 @@ +Param( + [Parameter(ValueFromRemainingArguments=$true)] + [string[]]$Args +) + +function Get-LatestDir($root) { + if (-not (Test-Path $root)) { return $null } + Get-ChildItem -Path $root -Directory -ErrorAction SilentlyContinue | + Where-Object { $_.Name -match '^[0-9]' } | + Sort-Object {[version]($_.Name)} -Descending | + Select-Object -First 1 -ExpandProperty FullName +} + +# Windows Kits (Program Files (x86)) +$kitsRoot = "${env:ProgramFiles(x86)}\Windows Kits\10\bin" +$kitsLatest = Get-LatestDir $kitsRoot +$kitsBinX64 = if ($kitsLatest) { Join-Path $kitsLatest 'x64' } else { $null } + +# MSVC (search common Program Files locations) +$msvcDirs = @() +$possibleRoots = @( + "${env:ProgramFiles}\Microsoft Visual Studio\2022\Community\VC\Tools\MSVC", + "${env:ProgramFiles(x86)}\Microsoft Visual Studio\2022\Community\VC\Tools\MSVC" +) +foreach ($pr in $possibleRoots) { + if (Test-Path $pr) { + $msvcDirs += Get-ChildItem -Path $pr -Directory -ErrorAction SilentlyContinue | Select-Object -ExpandProperty FullName + } +} +$msvcLatest = $null +if ($msvcDirs.Count -gt 0) { + $msvcLatest = $msvcDirs | Sort-Object { [version]((Split-Path $_ -Leaf)) } -Descending | Select-Object -First 1 +} +$msvcBin = if ($msvcLatest) { Join-Path $msvcLatest 'bin\Hostx64\x64' } else { $null } + +$added = @() +if ($msvcBin -and (Test-Path $msvcBin)) { $env:PATH = $msvcBin + ';' + $env:PATH; $added += $msvcBin } +if ($kitsBinX64 -and (Test-Path $kitsBinX64)) { $env:PATH = $kitsBinX64 + ';' + $env:PATH; $added += $kitsBinX64 } + +Write-Host "Prepended to PATH:" -ForegroundColor Cyan +if ($added.Count -eq 0) { Write-Host " (no toolchain bins found; building with current PATH)" -ForegroundColor Yellow } +foreach ($p in $added) { Write-Host " - $p" } + +# Execute prepare step then build (same sequence as existing package.json 'build') +try { + & node scripts/prepare-resources.cjs + if ($LASTEXITCODE -ne 0) { throw 'prepare-resources.cjs failed' } +} catch { + Write-Error "prepare-resources.cjs failed: $_" + exit 1 +} + +if (Get-Command bun -ErrorAction SilentlyContinue) { + & bun run build @Args + exit $LASTEXITCODE +} else { + & tauri build --config src/backend/tauri.conf.json @Args + exit $LASTEXITCODE +} diff --git a/.scripts/dump_gguf_specials.cjs b/.scripts/dump_gguf_specials.cjs new file mode 100644 index 0000000..c28b7d2 --- /dev/null +++ b/.scripts/dump_gguf_specials.cjs @@ -0,0 +1,28 @@ +const fs = require('fs'); +const p = process.argv[2]; +if (!p) { console.error('Usage: node dump_gguf_specials.cjs '); process.exit(2); } +const b = fs.readFileSync(p); +let s = b.toString('utf8'); +const keys = ['special_eog_ids','special_eos_id','special_tokens','tokenizer.ggml.tokens']; +for (const k of keys) { + const idx = s.indexOf(k); + if (idx===-1) { console.log(`${k}: NOT FOUND`); continue; } + const start = Math.max(0, idx-200), end = Math.min(s.length, idx+600); + console.log('='.repeat(40)); + console.log(`Key: ${k} at char ${idx}`); + console.log(s.slice(start,end)); +} +const m = s.match(/special_eog_ids\s*arr\[.*?\]\s*=\s*\[(.*?)\]/s); +if (m) console.log('parsed special_eog_ids:', m[1].trim()); else console.log('special_eog_ids pattern not parsed'); +const m2 = s.match(/special_eos_id\s*u32\s*=\s*(\d+)/); +if (m2) console.log('parsed special_eos_id:', m2[1]); else console.log('special_eos_id pattern not parsed'); +// Additional searches for tokens mentioned in logs +const toks = ['end_of_turn','','','eog','eog_ids']; +for (const t of toks) { + const idx = s.indexOf(t); + if (idx===-1) continue; + const start = Math.max(0, idx-80), end = Math.min(s.length, idx+80); + console.log('----'); + console.log(`Found token text: ${t} at ${idx}`); + console.log(s.slice(start,end)); +} diff --git a/.scripts/dump_gguf_specials.js b/.scripts/dump_gguf_specials.js new file mode 100644 index 0000000..d08c2c8 --- /dev/null +++ b/.scripts/dump_gguf_specials.js @@ -0,0 +1,18 @@ +const fs = require('fs'); +const p = process.argv[2]; +if (!p) { console.error('Usage: node dump_gguf_specials.js '); process.exit(2); } +const b = fs.readFileSync(p); +let s = b.toString('utf8'); +const keys = ['special_eog_ids','special_eos_id','special_tokens','tokenizer.ggml.tokens']; +for (const k of keys) { + const idx = s.indexOf(k); + if (idx===-1) { console.log(`${k}: NOT FOUND`); continue; } + const start = Math.max(0, idx-200), end = Math.min(s.length, idx+600); + console.log('='.repeat(40)); + console.log(`Key: ${k} at char ${idx}`); + console.log(s.slice(start,end)); +} +const m = s.match(/special_eog_ids\s*arr\[.*?\]\s*=\s*\[(.*?)\]/s); +if (m) console.log('parsed special_eog_ids:', m[1].trim()); else console.log('special_eog_ids pattern not parsed'); +const m2 = s.match(/special_eos_id\s*u32\s*=\s*(\d+)/); +if (m2) console.log('parsed special_eos_id:', m2[1]); else console.log('special_eos_id pattern not parsed'); diff --git a/.scripts/dump_gguf_specials.py b/.scripts/dump_gguf_specials.py new file mode 100644 index 0000000..577f257 --- /dev/null +++ b/.scripts/dump_gguf_specials.py @@ -0,0 +1,45 @@ +import sys +from pathlib import Path + +if len(sys.argv) < 2: + print('Usage: dump_gguf_specials.py ') + sys.exit(2) + +p = Path(sys.argv[1]) +if not p.exists(): + print('File not found:', p) + sys.exit(1) + +b = p.read_bytes() +try: + s = b.decode('utf-8', errors='replace') +except Exception as e: + s = ''.join(chr(c) if 32 <= c < 127 else '.' for c in b) + +keys = ['special_eos_id', 'special_eog_ids', 'special_tokens', 'tokenizer.ggml.tokens'] +for k in keys: + idx = s.find(k) + if idx == -1: + print(f"{k}: NOT FOUND") + continue + start = max(0, idx-200) + end = min(len(s), idx+600) + snippet = s[start:end] + print('='*40) + print(f'Key: {k} at byte {idx}') + print(snippet) + print() + +# Also attempt to find patterns like 'special_eog_ids arr' and numeric lists +import re +m = re.search(r'special_eog_ids\s*arr\[.*?\]\s*=\s*\[(.*?)\]', s, re.S) +if m: + print('parsed special_eog_ids:', m.group(1).strip()) +else: + print('special_eog_ids pattern not parsed') + +m2 = re.search(r'special_eos_id\s*u32\s*=\s*(\d+)', s) +if m2: + print('parsed special_eos_id:', m2.group(1)) +else: + print('special_eos_id pattern not parsed') diff --git a/.scripts/health_check.ps1 b/.scripts/health_check.ps1 new file mode 100644 index 0000000..0684f6b --- /dev/null +++ b/.scripts/health_check.ps1 @@ -0,0 +1,24 @@ +$urls = @( + 'http://127.0.0.1:8080/', + 'http://127.0.0.1:8080/health', + 'http://127.0.0.1:8080/status', + 'http://127.0.0.1:8080/api/health', + 'http://127.0.0.1:8080/v1/health' +) + +foreach ($u in $urls) { + Write-Host '---' + Write-Host 'URL:' $u + try { + $r = Invoke-WebRequest -Uri $u -UseBasicParsing -TimeoutSec 5 + Write-Host 'Status:' $r.StatusCode + if ($r.Content) { + $c = $r.Content + if ($c.Length -gt 1000) { $c = $c.Substring(0,1000) + '...[truncated]' } + Write-Host 'Body:' + Write-Host $c + } + } catch { + Write-Host 'Error:' $_.Exception.Message + } +} diff --git a/.scripts/run_and_check.ps1 b/.scripts/run_and_check.ps1 new file mode 100644 index 0000000..619d457 --- /dev/null +++ b/.scripts/run_and_check.ps1 @@ -0,0 +1,11 @@ +# Start telos-db, run health checks, show tail of log, then stop +Param() +Set-Location $PSScriptRoot.Replace('\.scripts','') +if (-not (Test-Path 'logs')) { New-Item -ItemType Directory -Path 'logs' | Out-Null } +$proc = Start-Process -FilePath .\target\debug\telos-db.exe -ArgumentList '--n_ctx','2048','--cache-ram','0' -RedirectStandardOutput .\logs\start.log -RedirectStandardError .\logs\start.err -NoNewWindow -PassThru +Start-Sleep -Seconds 6 +& .\.scripts\health_check.ps1 +Start-Sleep -Seconds 1 +Write-Host "--- Recent startup log ---" +Get-Content .\logs\start.log -Tail 200 | ForEach-Object { Write-Host $_ } +try { Stop-Process -Id $proc.Id -Force } catch { Write-Warning "Failed to stop process: $_" }