date              ✅
time              ✅
s-ip              ⚠ ignored by filter
cs-method         ✅
cs-uri-stem       ✅
cs-uri-query      ✅ (will be sanitized)
s-port            ⚠ ignored
cs-username       ❌ disable if enabled (we never want this)
c-ip              ⚠ hashed by filter, never raw
cs(User-Agent)    ✅
cs(Referer)       ✅
cs-host           ✅
sc-status         ✅
sc-bytes          ✅
time-taken        ⚠ optional, useful for performance, not transmitted

<#
  Maya — IIS LLM Bot Log Filter
  Reads W3C IIS logs, retains only verified LLM bot traffic, applies PII strip,
  produces NDJSON output.
 
  Usage:
    .\Export-LLMBotLogs.ps1 -LogPath 'C:\inetpub\logs\LogFiles\W3SVC1' `
                            -OutputPath 'D:\maya\export-2026-04-22.ndjson' `
                            -Since '2026-04-21T00:00:00Z' `
                            -TenantSecret (Get-MayaSecret)
#>
 
[CmdletBinding()]
param(
    [Parameter(Mandatory=$true)] [string]$LogPath,
    [Parameter(Mandatory=$true)] [string]$OutputPath,
    [Parameter(Mandatory=$true)] [datetime]$Since,
    [Parameter(Mandatory=$true)] [string]$TenantSecret,
    [switch]$DryRun
)
 
# 1. Verified LLM bot allowlist (case-insensitive substring match).
$BotPatterns = @(
    'GPTBot','ChatGPT-User','OAI-SearchBot',
    'ClaudeBot','Claude-Web','anthropic-ai',
    'PerplexityBot','Perplexity-User',
    'Google-Extended','Googlebot',
    'Bingbot',
    'Applebot-Extended',
    'Bytespider','CCBot',
    'Meta-ExternalAgent','FacebookBot',
    'DuckAssistBot','YouBot','Amazonbot','Diffbot'
)
 
# 2. Query-string keys to strip.
$DenyKeys = @('email','tckn','phone','customer_id','account_id',
              'session','token','key','secret','auth','csrf')
 
# 3. HMAC-SHA256 IP hasher.
$hmac = New-Object System.Security.Cryptography.HMACSHA256
$hmac.Key = [Text.Encoding]::UTF8.GetBytes($TenantSecret)
function Hash-IP([string]$ip) {
    if ([string]::IsNullOrWhiteSpace($ip)) { return $null }
    $bytes = $hmac.ComputeHash([Text.Encoding]::UTF8.GetBytes($ip))
    return [BitConverter]::ToString($bytes).Replace('-','').ToLower().Substring(0,32)
}
 
# 4. Sanitize query string.
function Sanitize-Query([string]$q) {
    if ([string]::IsNullOrWhiteSpace($q)) { return '' }
    $pairs = $q -split '&' | Where-Object { $_ -ne '' } | ForEach-Object {
        $kv = $_ -split '=', 2
        $k = $kv[0].ToLower()
        if ($DenyKeys -contains $k) { return $null }
        return $_
    } | Where-Object { $_ -ne $null }
    return ($pairs -join '&')
}
 
# 5. Discover log files modified since cutoff.
$logFiles = Get-ChildItem -Path $LogPath -Filter '*.log' -File |
    Where-Object { $_.LastWriteTimeUtc -ge $Since.ToUniversalTime() }
 
# 6. Iterate and emit NDJSON.
$writer = New-Object System.IO.StreamWriter($OutputPath, $false, [Text.Encoding]::UTF8)
$total = 0; $kept = 0
 
try {
    foreach ($file in $logFiles) {
        # Header detection — IIS W3C format.
        $headerLine = (Get-Content -Path $file.FullName -TotalCount 4 |
                       Where-Object { $_ -like '#Fields:*' }) -replace '#Fields:\s*',''
        if (-not $headerLine) { continue }
        $headers = $headerLine -split '\s+'
 
        Get-Content -Path $file.FullName | ForEach-Object {
            if ($_.StartsWith('#') -or [string]::IsNullOrWhiteSpace($_)) { return }
            $total++
            $values = $_ -split '\s+'
            if ($values.Length -ne $headers.Length) { return }
 
            $row = @{}
            for ($i = 0; $i -lt $headers.Length; $i++) {
                $row[$headers[$i]] = $values[$i]
            }
 
            $ua = [string]$row['cs(User-Agent)']
            $matched = $false
            foreach ($p in $BotPatterns) {
                if ($ua -match [regex]::Escape($p)) { $matched = $true; break }
            }
            if (-not $matched) { return }
 
            $stamp = "$($row['date'])T$($row['time'])Z"
            $stem = [string]$row['cs-uri-stem']
            $query = Sanitize-Query ([string]$row['cs-uri-query'])
            $path = if ($query) { "$stem`?$query" } else { $stem }
 
            $record = [ordered]@{
                timestamp        = $stamp
                user_agent       = $ua
                request_method   = [string]$row['cs-method']
                request_path     = $path
                status_code      = [int]([string]$row['sc-status'])
                response_bytes   = [int]([string]$row['sc-bytes'])
                referrer         = [string]$row['cs(Referer)']
                host             = [string]$row['cs-host']
                client_ip_hash   = Hash-IP ([string]$row['c-ip'])
            }
 
            $kept++
            if (-not $DryRun) {
                $writer.WriteLine( ($record | ConvertTo-Json -Compress) )
            }
        }
    }
}
finally {
    $writer.Flush(); $writer.Close()
    Write-Host "Scanned $total rows, retained $kept rows."
    if ($DryRun) { Write-Host 'Dry run — no file written.' }
}

{"timestamp":"2026-04-22T03:14:07Z","user_agent":"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.2; +https://openai.com/gptbot)","request_method":"GET","request_path":"/products/super-widget","status_code":200,"response_bytes":48329,"referrer":"","host":"www.example.com","client_ip_hash":"a4c6f1b2e7d8c1bb"}
{"timestamp":"2026-04-22T03:14:42Z","user_agent":"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; ClaudeBot/1.0; +https://www.anthropic.com/claudebot)","request_method":"GET","request_path":"/pricing","status_code":200,"response_bytes":62104,"referrer":"","host":"www.example.com","client_ip_hash":"7d9e22aa1f53e2c4"}
{"timestamp":"2026-04-22T03:15:11Z","user_agent":"Mozilla/5.0 (compatible; PerplexityBot/1.0; +https://perplexity.ai/perplexitybot)","request_method":"GET","request_path":"/products/super-widget","status_code":200,"response_bytes":48329,"referrer":"https://www.perplexity.ai/","host":"www.example.com","client_ip_hash":"3f8b22cc09111ab7"}

# 1) Each line is valid JSON
Get-Content export.ndjson | ForEach-Object { try { $null = $_ | ConvertFrom-Json } catch { Write-Host "Invalid: $_" } }
 
# 2) No disallowed fields appear (cookie, authorization, set-cookie, session, token)
Get-Content export.ndjson | Select-String -Pattern "(?i)(cookie|authorization|set-cookie|session|token|tckn|customer_id|account_id)"
# Expected output: nothing.
 
# 3) Every user_agent matches an LLM bot
Get-Content export.ndjson | ForEach-Object {
  $r = $_ | ConvertFrom-Json
  if ($r.user_agent -notmatch '(?i)GPTBot|ChatGPT-User|OAI-SearchBot|ClaudeBot|Claude-Web|anthropic-ai|PerplexityBot|Perplexity-User|Google-Extended|Googlebot|Bingbot|Applebot-Extended|Bytespider|CCBot|Meta-ExternalAgent|FacebookBot|DuckAssistBot|YouBot|Amazonbot|Diffbot') {
    Write-Host "Non-bot UA leaked: $($r.user_agent)"
  }
}
# Expected output: nothing.

$apiKey = $env:MAYA_API_KEY
$url    = 'https://ingest.withmaya.ai/v1/logs'
 
$response = Invoke-WebRequest -Uri $url `
    -Method Post `
    -InFile 'D:\maya\export-2026-04-22.ndjson' `
    -ContentType 'application/x-ndjson' `
    -Headers @{
        'Authorization' = "Bearer $apiKey"
        'X-Maya-Tenant' = 'acme-prod'
        'X-Maya-Schema' = '1'
    }
 
if ($response.StatusCode -ne 202) {
    throw "Maya ingestion returned $($response.StatusCode)"
}

schtasks /Create /TN "Maya LLM Log Export" /SC DAILY /ST 03:00 ^
  /TR "powershell.exe -ExecutionPolicy Bypass -File D:\maya\Export-LLMBotLogs.ps1 ..."