SecureChatAI.php

<?php

namespace Stanford\SecureChatAI;

require_once "emLoggerTrait.php";
require_once "classes/SecureChatLog.php";
require_once "classes/Models/ModelInterface.php";
require_once "classes/Models/BaseModelRequest.php";
require_once "classes/Models/GPTModelRequest.php";
require_once "classes/Models/WhisperModelRequest.php";
require_once "classes/Models/GeminiModelRequest.php";
require_once "classes/Models/ClaudeModelRequest.php";
require_once "classes/Models/GenericModelRequest.php";
require_once "classes/Models/GPT4oMiniTTSModelRequest.php";

require_once "classes/ToolInterface.php";
require_once "classes/ToolUse.php";
require_once "classes/ToolResult.php";
require_once "classes/ToolContext.php";
require_once "classes/AbortController.php";
require_once "classes/HookInterface.php";
require_once "classes/HookResult.php";
require_once "classes/ProjectAccessPreHook.php";
require_once "classes/JsonConfigToolAdapter.php";
require_once "classes/ToolPipeline.php";

require_once __DIR__ . '/vendor/autoload.php';

use Google\Exception;
use GuzzleHttp\Client;
use Yethee\Tiktoken\EncoderProvider;

class SecureChatAI extends \ExternalModules\AbstractExternalModule
{
    use emLoggerTrait;

    private array $defaultParams;
    private $guzzleClient = null;
    private $guzzleTimeout = 5.0;
    private array $modelConfig = [];
    private bool $initialized = false;
    private ?EncoderProvider $encoderProvider = null;

    public function __construct()
    {
        parent::__construct();
    }

    private function initSecureChatAI()
    {
        if ($this->initialized) return;
        // Set default LLM model parameters
        $this->defaultParams = [
            'temperature' => (float)$this->getSystemSetting('gpt-temperature') ?: 0.7,
            'top_p' => (float)$this->getSystemSetting('gpt-top-p') ?: 0.9,
            'frequency_penalty' => (float)$this->getSystemSetting('gpt-frequency-penalty') ?: 0.5,
            'presence_penalty' => (float)$this->getSystemSetting('gpt-presence-penalty') ?: 0,
            'max_tokens' => (int)$this->getSystemSetting('gpt-max-tokens') ?: 16384,
            'reasoning_effort' => $this->getSystemSetting('reasoning-effort'),
            'stop' => null,
            'model' => 'gpt-4o'
        ];

        // Initialize the model configurations from system settings
        $apiSettings = $this->framework->getSubSettings('api-settings');
        foreach ($apiSettings as $setting) {
            $modelAlias = trim($setting['model-alias'] ?? '');
            $modelID = trim($setting['model-id'] ?? '');
            $this->modelConfig[$modelAlias] = [
                'api_url' => trim($setting['api-url'] ?? ''),
                'api_token' => trim($setting['api-token'] ?? ''),
                'api_key_var' => trim($setting['api-key-var'] ?? ''),
                'required' => $setting['api-input-var'],
                'model_id' => $modelID
            ];
            if (isset($setting['default-model']) && $setting['default-model']) {
                $this->defaultParams['model'] = $modelAlias;
            }
        }

        // Set Guzzle info
        $timeout = $this->getSystemSetting('guzzle-timeout') ? (float)(strip_tags($this->getSystemSetting('guzzle-timeout'))) : $this->getGuzzleTimeout();
        $this->setGuzzleTimeout($timeout);
        $this->guzzleClient = $this->getGuzzleClient();

        $this->initialized = true;
    }

    public function getSecureChatLogs($offset)
    {
        $offset = intval($offset);
        return SecureChatLog::getAllLogs($this, $offset);
    }

    /**
     * Get logs for a specific session ID
     * @param string $session_id
     * @param int|null $project_id Optional project filter
     * @return array
     */
    public function getSecureChatLogsBySession($session_id, $project_id = null)
    {
        return SecureChatLog::getLogsBySession($this, $session_id, $project_id);
    }

    /**
     * Get logs for a specific project.
     * Fetches all logs and filters by the project_id stored in the JSON message payload,
     * since the EM framework's project_id column may not match (logs written from system context).
     * @param int $project_id
     * @param int $offset
     * @return array Decoded log data arrays (same format as SecureChatLog::getLog())
     */
    public function getProjectSecureChatLogs($project_id, $offset)
    {
        // Cannot use SecureChatLog or ASEMLO here because $module->queryLogs()
        // auto-filters by project_id when called from project context. Logs written
        // from system context have NULL in the EM project_id column even though the
        // JSON payload contains the correct project_id. We bypass the EM framework
        // entirely and load raw data.
        $pid = intval($project_id);
        $offset = intval($offset);
        $sql = "SELECT log_id, timestamp, record, message
                FROM redcap_external_modules_log
                WHERE external_module_id = (
                    SELECT external_module_id FROM redcap_external_modules
                    WHERE directory_prefix = ?
                    LIMIT 1
                )
                AND record IN ('SecureChatLog', 'SecureChatLogError')
                AND message LIKE ?
                ORDER BY log_id DESC
                LIMIT 1000 OFFSET ?";
        $pattern = '%"project_id":' . $pid . '%';
        $prefix = $this->PREFIX;
        $result = $this->query($sql, [$prefix, $pattern, $offset]);

        $logs = [];
        while ($row = $result->fetch_assoc()) {
            $decoded = json_decode($row['message'], true);
            if ($decoded === null) continue;
            // Verify project_id actually matches (LIKE is broad)
            $logPid = $decoded['project_id'] ?? null;
            if (intval($logPid) !== $pid) continue;
            $decoded['timestamp'] = $row['timestamp'];
            $decoded['id'] = $row['log_id'];
            $decoded['record'] = $row['record'];
            $logs[] = $decoded;
        }
        return $logs;
    }

    /**
     * Rehydrate a session by session_id using raw SQL (bypasses EM framework project scoping).
     * @param string $session_id
     * @param int $project_id
     * @return array Session object with 'messages', 'metadata', 'stats'
     */
    public function rehydrateProjectSession($session_id, $project_id, $username = null)
    {
        $pid = intval($project_id);
        $prefix = $this->PREFIX;

        // --- Preferred path: dedicated one-row-per-turn records ---
        // CappyChatTurn / CappyChatTurnError rows (written by logConversationTurn)
        // are already clean user-prompt + final-answer pairs, so rebuilding from
        // them is deterministic. Deriving turns from the per-LLM-call audit log
        // is lossy in agent mode (N rows/turn, injected TOOL RESULT messages and
        // tool-call-only envelopes leak in, errored turns vanish).
        $turnSql = "SELECT l.log_id, l.timestamp, l.record, l.message
                    FROM redcap_external_modules_log l
                    JOIN redcap_external_modules_log_parameters p
                        ON l.log_id = p.log_id AND p.name = 'session_id' AND p.value = ?
                    WHERE l.external_module_id = (
                        SELECT external_module_id FROM redcap_external_modules
                        WHERE directory_prefix = ? LIMIT 1
                    )
                    AND l.record IN ('CappyChatTurn', 'CappyChatTurnError')
                    ORDER BY l.log_id ASC";
        $turnResult = $this->query($turnSql, [$session_id, $prefix]);
        $turnLogs = [];
        $firstTurnLogId = null;
        while ($row = $turnResult->fetch_assoc()) {
            $decoded = json_decode($row['message'], true);
            if ($decoded === null) continue;
            if (intval($decoded['project_id'] ?? 0) !== $pid) continue;
            // Authorization: session ids are client-generated and guessable, so
            // scope to the requesting user. Turn rows carry the username that
            // created them; when a user is supplied, only that user's turns are
            // returned. Prevents rebuilding another user's chat (possible PHI)
            // from a guessed session_id within the same project.
            if ($username !== null && (string)($decoded['username'] ?? '') !== (string)$username) continue;
            $decoded['timestamp'] = $row['timestamp'];
            $decoded['id'] = $row['log_id'];
            $decoded['record'] = $row['record'];
            if ($firstTurnLogId === null) $firstTurnLogId = (int)$row['log_id'];
            $turnLogs[] = $decoded;
        }

        if (!empty($turnLogs)) {
            $session = $this->buildSessionFromTurns($session_id, $pid, $turnLogs);

            // Sessions active across the turn-logging deploy have their earlier
            // turns only as legacy audit rows. Merge those (rows before the first
            // turn row) so cross-deploy history isn't lost. Only when the request
            // is NOT user-scoped: legacy rows carry no username, so ownership is
            // unverifiable and they must not be exposed on a user-scoped request.
            if ($username === null && $firstTurnLogId !== null) {
                $legacy = $this->legacyReconstructSession($session_id, $pid, $prefix, $firstTurnLogId);
                if (!empty($legacy['messages'])) {
                    $session['messages'] = array_merge($legacy['messages'], $session['messages']);
                    $session['stats']['total_turns'] += ($legacy['stats']['total_turns'] ?? 0);
                    $session['stats']['total_tokens'] += ($legacy['stats']['total_tokens'] ?? 0);
                    $session['stats']['models_used'] = array_values(array_unique(array_merge(
                        $legacy['stats']['models_used'] ?? [],
                        $session['stats']['models_used'] ?? []
                    )));
                    if (!empty($legacy['metadata']['start_time'])) {
                        $session['metadata']['start_time'] = $legacy['metadata']['start_time'];
                    }
                }
            }
            return $session;
        }

        // No turn rows. A user-scoped request cannot be satisfied from legacy
        // audit rows — they carry no username, so ownership is unverifiable.
        // Deny rather than risk exposing another user's pre-deploy history.
        if ($username !== null) {
            return [
                'session_id' => $session_id,
                'messages' => [],
                'metadata' => [],
                'stats' => ['total_turns' => 0, 'total_tokens' => 0, 'models_used' => []]
            ];
        }

        // Unscoped fallback: reconstruct from the per-LLM-call audit log.
        return $this->legacyReconstructSession($session_id, $pid, $prefix);
    }

    /**
     * Legacy rehydration from the per-LLM-call audit log (SecureChatLog rows).
     * Used for sessions that predate turn logging, and to backfill the pre-turn
     * portion of sessions that straddle the deploy. $maxLogIdExclusive, when
     * given, limits the scan to rows strictly before that log_id.
     */
    private function legacyReconstructSession($session_id, $pid, $prefix, $maxLogIdExclusive = null)
    {
        $bound = $maxLogIdExclusive !== null ? " AND l.log_id < ?" : "";

        $sql = "SELECT l.log_id, l.timestamp, l.record, l.message
                FROM redcap_external_modules_log l
                JOIN redcap_external_modules_log_parameters p
                    ON l.log_id = p.log_id AND p.name = 'session_id' AND p.value = ?
                WHERE l.external_module_id = (
                    SELECT external_module_id FROM redcap_external_modules
                    WHERE directory_prefix = ? LIMIT 1
                )
                AND l.record IN ('SecureChatLog', 'SecureChatLogError')$bound
                ORDER BY l.log_id ASC";
        $params = $maxLogIdExclusive !== null
            ? [$session_id, $prefix, $maxLogIdExclusive]
            : [$session_id, $prefix];
        $result = $this->query($sql, $params);

        $logs = [];
        while ($row = $result->fetch_assoc()) {
            $decoded = json_decode($row['message'], true);
            if ($decoded === null) continue;
            if (intval($decoded['project_id'] ?? 0) !== $pid) continue;
            $decoded['timestamp'] = $row['timestamp'];
            $decoded['id'] = $row['log_id'];
            $decoded['record'] = $row['record'];
            $logs[] = $decoded;
        }

        // Fallback: scan JSON message for pre-promotion logs
        if (empty($logs)) {
            $bound2 = $maxLogIdExclusive !== null ? " AND log_id < ?" : "";
            $sql2 = "SELECT log_id, timestamp, record, message
                     FROM redcap_external_modules_log
                     WHERE external_module_id = (
                         SELECT external_module_id FROM redcap_external_modules
                         WHERE directory_prefix = ? LIMIT 1
                     )
                     AND record IN ('SecureChatLog', 'SecureChatLogError')
                     AND message LIKE ?$bound2
                     ORDER BY log_id ASC
                     LIMIT 1000";
            $pattern = '%"session_id":"' . preg_replace('/[^a-zA-Z0-9_-]/', '', $session_id) . '"%';
            $params2 = $maxLogIdExclusive !== null
                ? [$prefix, $pattern, $maxLogIdExclusive]
                : [$prefix, $pattern];
            $result2 = $this->query($sql2, $params2);
            while ($row = $result2->fetch_assoc()) {
                $decoded = json_decode($row['message'], true);
                if ($decoded === null) continue;
                if (intval($decoded['project_id'] ?? 0) !== $pid) continue;
                if (($decoded['session_id'] ?? '') !== $session_id) continue;
                $decoded['timestamp'] = $row['timestamp'];
                $decoded['id'] = $row['log_id'];
                $decoded['record'] = $row['record'];
                $logs[] = $decoded;
            }
        }

        if (empty($logs)) {
            return [
                'session_id' => $session_id,
                'messages' => [],
                'metadata' => [],
                'stats' => ['total_turns' => 0, 'total_tokens' => 0, 'models_used' => []]
            ];
        }

        $messages = [];
        $models = [];
        $total_tokens = 0;
        $first_timestamp = null;
        $last_timestamp = null;

        foreach ($logs as $index => $log) {
            if (!empty($log['error'])) continue;

            if (!empty($log['user_message'])) {
                $messages[] = [
                    'role' => 'user',
                    'content' => $log['user_message'],
                    'turn' => $index + 1
                ];
            }
            if (!empty($log['assistant_response'])) {
                $messages[] = [
                    'role' => 'assistant',
                    'content' => $log['assistant_response'],
                    'turn' => $index + 1,
                    'tools_used' => $log['tools_used'] ?? null
                ];
            }
            if (!empty($log['model'])) $models[$log['model']] = true;
            if (!empty($log['usage']['total_tokens'])) $total_tokens += $log['usage']['total_tokens'];
            if ($first_timestamp === null) $first_timestamp = $log['timestamp'];
            $last_timestamp = $log['timestamp'];
        }

        return [
            'session_id' => $session_id,
            'messages' => $messages,
            'metadata' => [
                'project_id' => $pid,
                'start_time' => $first_timestamp,
                'end_time' => $last_timestamp,
                'duration_seconds' => $first_timestamp && $last_timestamp ?
                    strtotime($last_timestamp) - strtotime($first_timestamp) : 0
            ],
            'stats' => [
                'total_turns' => count($messages) / 2,
                'total_tokens' => $total_tokens,
                'models_used' => array_keys($models)
            ]
        ];
    }

    private function filterDefaultParamsForModel($model, $params)
    {
        // Embedding models don't use chat defaultParams
        if ($model === 'ada-002' || $model === 'text-embedding-3-small' || $model === 'text-embedding-3-large') {
            $embedParams = array_merge([
                'model' => $this->modelConfig[$model]['model_id'] ?? $model
            ], $params);
            unset($embedParams['session_id'], $embedParams['agent_mode'], $embedParams['project_id'], $embedParams['log_turn']);
            return $embedParams;
        }

        $merged = array_merge($this->defaultParams, $params);

        // Only o1/o3-mini/gpt-5 get reasoning params
        if (!in_array($model, ['o1', 'o3-mini', 'gpt-5'])) {
            unset($merged['reasoning']);
            unset($merged['reasoning_effort']);
        }

        // Only models supporting json_schema
        $schemaModels = ['gpt-4-1', 'gpt-4-1-nano', 'gpt-5', 'gpt-5-4', 'gpt-5-4-nano', 'o1', 'o3', 'o3-mini', 'o4-mini'];
        if (!in_array($model, $schemaModels)) {
            unset($merged['json_schema']);
        }

        // Only reasoning models have strict param set (use max_completion_tokens)
        if (in_array($model, ['o1', 'o3-mini', 'o3', 'o4-mini', 'gpt-5'])) {
            $strict = [
                'model' => $model,
                'messages' => $merged['messages'] ?? [],
                'max_completion_tokens' => $merged['max_completion_tokens'] ?? ($merged['max_tokens'] ?? 32000),
            ];
            if (isset($merged['reasoning_effort'])) {
                $strict['reasoning_effort'] = $merged['reasoning_effort'];
            }

            // Preserve json_schema for reasoning models
            if (isset($merged['json_schema'])) {
                $strict['json_schema'] = $merged['json_schema'];
            }
            return $strict;
        }

        // Remove max_tokens for all non-o1/o3-mini
        unset($merged['max_tokens']);

        // Strip internal/meta params that are not part of the API contract
        unset($merged['session_id'], $merged['agent_mode'], $merged['project_id'], $merged['log_turn']);

        return $merged;
    }

    public function callAI($model, $params = [], $project_id = null, $username = null)
    {
        $startTotal = microtime(true);
        $retries = 2;
        $attempt = 0;

        while ($attempt <= $retries) {
            try {
                $startAttempt = microtime(true);

                // --- Server-Side Compaction (opt-in, before agent mode gate) ---
                $params = $this->compactMessagesIfNeeded($params, $model);

                // --- Agent Mode Gate ---

                $agent_mode_requested = !empty($params['agent_mode']);
                $agent_mode_enabled = (bool) $this->getSystemSetting('enable_agent_mode');

                $response = null;

                if ($agent_mode_requested && $agent_mode_enabled) {
                    $startAgentLoop = microtime(true);
                    $response = $this->runAgentLoop(
                        model: $model,
                        params: $params,
                        project_id: $project_id,
                        username: $username
                    );
                    $this->emDebug("callAI timing - runAgentLoop", [
                        'duration_ms' => round((microtime(true) - $startAgentLoop) * 1000, 2)
                    ]);
                } else {
                    if($agent_mode_requested && !$agent_mode_enabled){
                        $this->emDebug("Agent mode requested but not enabled in system settings. Proceeding with normal LLM call.");
                        unset($params['agent_mode']);
                    }

                    // Normal single-call path
                    $startLLMCall = microtime(true);
                    $response = $this->callLLMOnce($model, $params, $project_id);
                    $this->emDebug("callAI timing - callLLMOnce", [
                        'duration_ms' => round((microtime(true) - $startLLMCall) * 1000, 2)
                    ]);
                }

                // ✅ ALWAYS sanitize output before returning to UI
                $startSanitize = microtime(true);
                $sanitizedResponse = $this->sanitizeOutputForUI($response);
                $this->emDebug("callAI timing - sanitizeOutputForUI", [
                    'duration_ms' => round((microtime(true) - $startSanitize) * 1000, 2)
                ]);

                $this->emDebug("callAI timing - attempt complete", [
                    'attempt' => $attempt + 1,
                    'attempt_duration_ms' => round((microtime(true) - $startAttempt) * 1000, 2),
                    'total_duration_ms' => round((microtime(true) - $startTotal) * 1000, 2)
                ]);

                // One clean turn record for reliable rehydration. Log the
                // sanitized content (what the user actually saw), but carry over
                // tools_used/usage from the pre-sanitize response.
                $turnResponse = is_array($sanitizedResponse) ? $sanitizedResponse : ['content' => (string)$sanitizedResponse];
                if (empty($turnResponse['tools_used']) && !empty($response['tools_used'])) {
                    $turnResponse['tools_used'] = $response['tools_used'];
                }
                if (empty($turnResponse['usage']) && !empty($response['usage'])) {
                    $turnResponse['usage'] = $response['usage'];
                }
                if (!empty($response['error'])) {
                    $turnResponse['error'] = true;
                }
                $this->logConversationTurn($project_id, $params, $turnResponse, $username);

                return $sanitizedResponse;

            } catch (\Exception $e) {
                $attempt++;
                // PHI-safe: log the error description and request metadata only.
                // Never log $params directly — it holds the prompt messages (PHI).
                $this->emDebug("Attempt $attempt failed", [
                    'model'         => $model,
                    'error'         => $e->getMessage(),
                    'message_count' => isset($params['messages']) && is_array($params['messages'])
                        ? count($params['messages'])
                        : 0,
                ]);

                if ($attempt > $retries) {
                    $error = [
                        'error' => true,
                        'type' => 'NETWORK_ERROR',
                        'message' => "Error after $retries retries: " . $e->getMessage()
                    ];
                    if ($project_id) {
                        $this->logErrorInteraction($project_id, $params, $error);
                    } else {
                        $this->emDebug("Skipping error logging due to missing project ID (pid).");
                    }

                    $this->emDebug("callAI timing - failed after retries", [
                        'total_duration_ms' => round((microtime(true) - $startTotal) * 1000, 2)
                    ]);

                    // ✅ Sanitize errors too
                    $sanitizedError = $this->sanitizeOutputForUI($error);

                    // Turn record so the user's question survives rehydration
                    // even when the request failed outright (no answer produced).
                    $turnError = is_array($sanitizedError) ? $sanitizedError : ['content' => (string)$sanitizedError];
                    $turnError['error'] = true;
                    if (empty($turnError['model'])) {
                        $turnError['model'] = $model;
                    }
                    $this->logConversationTurn($project_id, $params, $turnError, $username);

                    return $sanitizedError;
                }
            }
        }

        $this->emDebug("callAI timing - unknown error fallback", [
            'total_duration_ms' => round((microtime(true) - $startTotal) * 1000, 2)
        ]);

        // ✅ Sanitize fallback error
        return $this->sanitizeOutputForUI([
            'error' => true,
            'type' => 'UNKNOWN_ERROR',
            'message' => 'Unknown error'
        ]);
    }

    // -------------------------------------------------------------------------
    // Public inspection helpers (for test/diagnostic EMs)
    // -------------------------------------------------------------------------

    /**
     * Return the full tool catalog for a project (EM-discovered + JSON registry).
     */
    public function getToolCatalogForProject(?int $pid): array
    {
        return $this->loadToolsForProject($pid);
    }

    /**
     * Return the list of configured model aliases.
     */
    public function getAvailableModels(): array
    {
        $this->initSecureChatAI();
        return array_keys($this->modelConfig);
    }

    /**
     * Return the context window size (in tokens) for a model.
     */
    public function getModelContextWindow(string $model): int
    {
        return $this->getModelContextSpec($model)['context'] ?? 128000;
    }

    /**
     * Run server-side message compaction and return before/after stats.
     *
     * Makes one LLM call (to generate the summary) if compaction triggers.
     */
    public function runCompaction(array $messages, string $model, int $keepRecent = 6, ?string $summarizeModel = null, ?int $contextWindowOverride = null): array
    {
        $params = [
            'messages'             => $messages,
            'compact_if_needed'    => true,
            'compact_keep_recent'  => $keepRecent,
            'compact_model'        => $summarizeModel ?? $model,
        ];

        $beforeCount = count($messages);
        $beforeText  = $this->messagesToText($messages);
        $beforeTokens = $this->estimateTokens($beforeText, $model);

        $result = $this->compactMessagesIfNeeded($params, $model, $contextWindowOverride);

        $afterMessages = $result['messages'] ?? $messages;
        $afterCount    = count($afterMessages);
        $afterText     = $this->messagesToText($afterMessages);
        $afterTokens   = $this->estimateTokens($afterText, $model);

        $contextWindow = $contextWindowOverride ?? $this->getModelContextWindow($model);
        $threshold     = (int)($contextWindow * 0.8);

        return [
            'compacted'       => ($afterCount < $beforeCount),
            'context_window'  => $contextWindow,
            'threshold'       => $threshold,
            'before_messages' => $beforeCount,
            'before_tokens'   => $beforeTokens,
            'after_messages'  => $afterCount,
            'after_tokens'    => $afterTokens,
            'reduction_pct'   => $beforeTokens > 0
                ? round((1 - $afterTokens / $beforeTokens) * 100, 1)
                : 0,
            'messages'        => $afterMessages,
        ];
    }

    private function loadToolsForProject(?int $pid): array
    {
        if (empty($pid)) return [];

        // Discover tool definitions from enabled EMs (project-level overrides system-level)
        $emTools = $this->discoverEmToolDefinitions($pid);

        // Validate each tool definition
        $validatedTools = [];
        foreach ($emTools as $tool) {
            $validation = $this->validateToolDefinition($tool);
            if ($validation['valid']) {
                $validatedTools[] = $tool;
            } else {
                $this->emError("Invalid tool definition skipped", [
                    'tool_name' => $tool['name'] ?? 'unnamed',
                    'errors' => $validation['errors']
                ]);
            }
        }

        return $validatedTools;
    }

    /**
     * Discover tool definitions from enabled tool EMs.
     *
     * Reads agent-tool-definitions from config.json of EMs listed
     * in the agent_tool_em_prefixes setting.
     *
     * Project-level setting takes priority; falls back to system-level.
     *
     * @param int|null $pid Project ID for project-level prefix lookup
     * @return array Tool configs with auto-filled endpoint/module fields
     */
    private function discoverEmToolDefinitions(?int $pid = null): array
    {
        // Union of system-level (everyone gets) + project-level (this project only)
        $systemRaw  = $this->getSystemSetting('agent_tool_em_prefixes') ?? '';
        $projectRaw = (!empty($pid)) ? ($this->getProjectSetting('project_agent_tool_em_prefixes', $pid) ?? '') : '';

        $systemPrefixes  = $systemRaw  ? array_map('trim', explode(',', $systemRaw))  : [];
        $projectPrefixes = $projectRaw ? array_map('trim', explode(',', $projectRaw)) : [];
        $prefixes = array_values(array_unique(array_filter(array_merge($systemPrefixes, $projectPrefixes))));
        $tools = [];

        foreach ($prefixes as $prefix) {
            if (empty($prefix)) continue;

            try {
                $moduleDir = $this->getModuleDirectoryPath($prefix);

                // Read tool definitions from tools.json manifest
                $toolsPath = $moduleDir . '/tools.json';
                if (!file_exists($toolsPath)) {
                    $this->emDebug("tools.json not found for tool EM: {$prefix}");
                    continue;
                }

                $manifest = json_decode(file_get_contents($toolsPath), true);
                if (json_last_error() !== JSON_ERROR_NONE) {
                    $this->emError("Invalid tools.json for tool EM: {$prefix}");
                    continue;
                }

                $definitions = $manifest['tools'] ?? [];
                foreach ($definitions as $def) {
                    $action = $def['action'] ?? '';
                    $tools[] = [
                        'name'        => $def['name'] ?? '',
                        'description' => $def['description'] ?? '',
                        'endpoint'    => 'module_api',
                        'module'      => [
                            'prefix' => $prefix,
                            'action' => $action,
                        ],
                        'parameters'  => $def['parameters'] ?? ['type' => 'object', 'properties' => []],
                        'readOnly'    => $def['readOnly'] ?? false,
                        'destructive' => $def['destructive'] ?? false,
                    ];
                }

            } catch (\Throwable $e) {
                $this->emError("Failed to discover tools from {$prefix}: " . $e->getMessage());
            }
        }

        return $tools;
    }

    /**
     * Get the filesystem path for a REDCap external module by prefix.
     */
    private function getModuleDirectoryPath(string $prefix): string
    {
        if (method_exists('\ExternalModules\ExternalModules', 'getModuleDirectoryPath')) {
            $path = \ExternalModules\ExternalModules::getModuleDirectoryPath($prefix);
            if ($path !== false) {
                return $path;
            }
        }

        // Fallback: check both modules/ and modules-local/
        $docRoot = APP_PATH_DOCROOT . "../";
        foreach (['modules/', 'modules-local/'] as $dir) {
            $pattern = $docRoot . $dir . $prefix . "_v*";
            $matches = glob($pattern, GLOB_ONLYDIR);
            if (!empty($matches)) {
                return $matches[0];
            }
        }

        return $docRoot . "modules/{$prefix}";
    }

    /**
     * Validate tool definition structure
     * Returns ['valid' => bool, 'errors' => array]
     */
    private function validateToolDefinition(array $tool): array
    {
        $errors = [];

        // Required fields
        if (empty($tool['name'])) {
            $errors[] = "Missing 'name' field";
        } elseif (!preg_match('/^[a-zA-Z][a-zA-Z0-9_.]*$/', $tool['name'])) {
            $errors[] = "Invalid 'name' format (must start with letter, alphanumeric + _ . only)";
        }

        if (empty($tool['description'])) {
            $errors[] = "Missing 'description' field";
        }

        if (empty($tool['endpoint'])) {
            $errors[] = "Missing 'endpoint' field";
        } elseif (!in_array($tool['endpoint'], ['module_api', 'redcap_api', 'http'])) {
            $errors[] = "Invalid 'endpoint' value (must be: module_api, redcap_api, or http)";
        }

        // Validate parameters structure
        if (isset($tool['parameters'])) {
            if (!is_array($tool['parameters'])) {
                $errors[] = "'parameters' must be an object";
            } elseif (!isset($tool['parameters']['type']) || $tool['parameters']['type'] !== 'object') {
                $errors[] = "'parameters.type' must be 'object'";
            }
        }

        // Validate endpoint-specific requirements
        if ($tool['endpoint'] === 'redcap_api') {
            if (empty($tool['redcap']['prefix'])) {
                $errors[] = "Missing 'redcap.prefix' for redcap_api endpoint";
            }
            if (empty($tool['redcap']['action'])) {
                $errors[] = "Missing 'redcap.action' for redcap_api endpoint";
            }
        } elseif ($tool['endpoint'] === 'module_api') {
            if (empty($tool['module']['action'])) {
                $errors[] = "Missing 'module.action' for module_api endpoint";
            }
        }

        return [
            'valid' => empty($errors),
            'errors' => $errors
        ];
    }

    private function getAgentRouterPrompt(): string
    {
        $prompt = (string) ($this->getSystemSetting('agent_router_system_prompt') ?? '');
        if (!empty(trim($prompt))) return $prompt;

        // Safe default if not configured yet
        return implode("\n", [
            "You are the REDCap Agent Router.",
            "Your job:",
            "- Interpret user intent.",
            "- Decide if a tool is required.",
            "- Choose the correct tool(s) and provide a tool_call in strict JSON.",
            "- Ask for missing required parameters instead of guessing.",
            "- Never invent tool names or arguments.",
            "",
            "Output one of:",
            "1) A tool_call JSON object",
            "2) A clarification question",
            "3) A final natural-language answer"
        ]);
    }

    private function buildToolCatalogText(array $tools): string
    {
        if (empty($tools)) {
            return "TOOLS AVAILABLE: (none)";
        }

        $lines = ["TOOLS AVAILABLE:"];
        foreach ($tools as $t) {
            $name = $t['name'] ?? '(missing name)';
            $desc = $t['description'] ?? '';
            $required = $t['parameters']['required'] ?? [];
            $reqStr = !empty($required) ? implode(", ", $required) : "(none)";
            $lines[] = "- {$name}: {$desc} | required: {$reqStr}";
        }
        return implode("\n", $lines);
    }

    private function injectAgentSystemContext(array $messages, array $tools, int $depth = 0, int $maxDepth = 1): array
    {
        $routerPrompt = $this->getAgentRouterPrompt();

        // Add spawnAgent tool to catalog when depth allows
        if ($depth < $maxDepth) {
            $spawnTool = [
                'name'        => 'spawnAgent',
                'description' => 'Spawn a sub-agent to handle a task independently. '
                    . 'The sub-agent has its own tool pool and returns a final result.',
                'parameters'  => [
                    'type'       => 'object',
                    'properties' => [
                        'prompt' => [
                            'type'        => 'string',
                            'description' => 'Task for the sub-agent. Be specific and self-contained.'
                        ],
                        'allowed_tools' => [
                            'type'        => 'array',
                            'items'       => ['type' => 'string'],
                            'description' => 'Subset of tools the sub-agent can use. Empty = all tools.'
                        ]
                    ],
                    'required' => ['prompt']
                ]
            ];
            $tools[] = $spawnTool;
        }

        // ALWAYS include tool catalog in agent mode
        $routerPrompt .= "\n\n" . $this->buildToolCatalogText($tools);

        array_unshift($messages, [
            'role' => 'system',
            'content' => $routerPrompt
        ]);

        return $messages;
    }

    private function runAgentLoop(string $model, array $params, ?int $project_id, ?string $username = null, int $depth = 0, int $maxDepth = 1): array
    {
        $messages = $params['messages'] ?? [];
        $tools = $this->loadToolsForProject($project_id);

        // Force schema-capable model for agent mode. Pick the fallback
        // dynamically from the *configured* schema-capable models so we never
        // silently switch to an alias that isn't in $this->modelConfig (which
        // would throw "Unsupported model" mid-loop and surface as a generic
        // "network difficulties" error to the user).
        $schemaModels = ['gpt-4-1', 'gpt-4-1-nano', 'gpt-5', 'gpt-5-4', 'gpt-5-4-nano', 'o1', 'o3', 'o3-mini', 'o4-mini'];
        if (!in_array($model, $schemaModels)) {
            $configured = array_keys($this->modelConfig);
            $fallback = null;
            foreach ($schemaModels as $candidate) {
                if (in_array($candidate, $configured, true)) {
                    $fallback = $candidate;
                    break;
                }
            }
            if ($fallback === null) {
                throw new \Exception(
                    "Agent mode requires a schema-capable model but none of [" . implode(', ', $schemaModels)
                    . "] are configured in SecureChatAI's api-settings. Add one, or disable agent mode for this project."
                );
            }
            $this->emDebug("Agent mode requires schema-capable model, switching from {$model} to {$fallback}");
            $model = $fallback;
        }

        $this->emDebug("AGENT MODE ENABLED", [
            'pid' => $project_id,
            'model' => $model,
            'tool_count' => count($tools),
            'tool_names' => array_column($tools, 'name')
        ]);

        // Initialize safety limits (reduced for sub-agents)
        $max_steps = (int) ($this->getSystemSetting('agent_max_steps') ?? 8);
        $max_tools = (int) ($this->getSystemSetting('agent_max_tools_per_run') ?? 15);
        $timeout = (int) ($this->getSystemSetting('agent_timeout_seconds') ?? 120);
        $max_tool_result_chars = (int) ($this->getSystemSetting('agent_max_tool_result_chars') ?? 8000);

        // Sub-agents get reduced limits to prevent runaway token usage
        if ($depth > 0) {
            $max_steps = max(2, (int) ($max_steps / 2));
            $max_tools = max(2, (int) ($max_tools / 2));
            $timeout = max(15, (int) ($timeout / 2));
        }
        $start_time = time();

        // CRITICAL: Agent mode needs higher token limit to avoid truncation mid-JSON
        // Reasoning models use max_completion_tokens, others use max_tokens
        if (in_array($model, ['o1', 'o3-mini', 'o3', 'o4-mini'])) {
            $params['max_completion_tokens'] = 32000;
        } else {
            $params['max_tokens'] = 4000;
        }

        $step = 0;
        $tools_called = 0;
        $tool_call_history = []; // Track tool calls to detect loops
        $tools_used = []; // Track tools used for UI display

        // Inject router system prompt + tool catalog
        $messages = $this->injectAgentSystemContext($messages, $tools, $depth, $maxDepth);
        $params['messages'] = $messages;

        // ⚠️ IMPORTANT: disable native OpenAI tools for agent mode
        unset($params['tools'], $params['tool_choice']);

        // Prevent recursion
        unset($params['agent_mode']);

        // Add JSON schema for agent responses
        // Note: strict=false because tool arguments are dynamic (incompatible with strict mode)
        $params['json_schema'] = [
            'name' => 'agent_response',
            'strict' => false,
            'schema' => [
                'type' => 'object',
                'properties' => [
                    'tool_call' => [
                        'type' => 'object',
                        'properties' => [
                            'name' => ['type' => 'string'],
                            'arguments' => ['type' => 'object']
                        ],
                        'required' => ['name', 'arguments'],
                        'additionalProperties' => false
                    ],
                    'final_answer' => ['type' => 'string'],
                    'thinking' => ['type' => 'string']
                ],
                'additionalProperties' => false
            ]
        ];

        while ($step < $max_steps) {
            $step++;

            // Step-by-step logging (PHI-safe: log only metadata, not message content)
            $this->emDebug("AGENT STEP {$step}/{$max_steps}", [
                'messages_count' => count($messages),
                'last_message_length' => strlen(end($messages)['content'] ?? '')
            ]);

            $response = $this->callLLMOnce($model, $params, $project_id);
            // PHI-safe: log only response metadata (token usage, model, shape) — never the
            // raw content body, which can contain prompt/response PHI.
            $this->emDebug("AGENT RAW RESPONSE (metadata only)", $this->summarizeResponseForDebug($response));

            $content = trim($response['content'] ?? '');

            // Decode HTML entities (REDCap may encode response)
            $content = html_entity_decode($content, ENT_QUOTES, 'UTF-8');

            // Clean ALL control characters (including literal newlines/tabs) that break JSON parsing
            // This collapses pretty-printed JSON into single line
            // Escaped sequences like \n are preserved since they're not actual control chars
            $cleanContent = preg_replace('/[\x00-\x1F]/', '', $content);

            $decoded = json_decode($cleanContent, true);

            // Fallback strategy for non-JSON responses
            if (json_last_error() !== JSON_ERROR_NONE) {
                $this->emDebug("JSON parse failed, trying regex extraction", [
                    'error' => json_last_error_msg(),
                    'content_preview' => substr($content, 0, 200)
                ]);

                // Try to extract tool_call JSON pattern from text
                if (preg_match('/\{[\s\S]*"tool_call"[\s\S]*\}/', $content, $matches)) {
                    $decoded = json_decode($matches[0], true);
                    if (json_last_error() === JSON_ERROR_NONE) {
                        $this->emDebug("Regex extraction succeeded");
                    }
                }

                // If still no valid JSON, treat as plain text final answer
                if (!is_array($decoded) || json_last_error() !== JSON_ERROR_NONE) {
                    $this->emDebug("Treating plain text response as final answer");
                    return [
                        'role' => 'assistant',
                        'content' => $content
                    ];
                }
            }

            if (isset($decoded['tool_call'])) {
                $tool_call = $decoded['tool_call'];
                $tool_name = $tool_call['name'] ?? null;
                $arguments = $tool_call['arguments'] ?? [];

                if (!$tool_name) {
                    return $this->agentError("INVALID_TOOL_CALL", "Tool name missing");
                }

                $execution = $this->executeToolCall(
                    tool_name: $tool_name,
                    arguments: $arguments,
                    tools: $tools,
                    project_id: $project_id,
                    model: $model,
                    username: $username
                );

                // Return errors immediately (but not MISSING_PARAMETERS - let agent handle that)
                if ($execution['error'] ?? false) {
                    return $execution;
                }

                // Cap tool result size to prevent token overflow
                $execution['result'] = $this->capToolResultSize(
                    $execution['result'],
                    $max_tool_result_chars
                );

                // Track tool usage for UI display. For records.search results
                // that carry a cache reference, also pass paging metadata
                // through so the chat UI can render its own pagination
                // controls against the server-side cache — the LLM shouldn't
                // burn turns clicking "next page".
                $usageEntry = [
                    'name' => $tool_name,
                    'arguments' => $arguments,
                    'step' => $step
                ];
                if ($tool_name === 'records.search' && is_array($execution['result'])) {
                    $r = $execution['result'];
                    if (!empty($r['reference']) && isset($r['total_record_count'])) {
                        $usageEntry['paging'] = [
                            'reference' => $r['reference'],
                            'total' => (int)$r['total_record_count'],
                            'offset' => (int)($r['offset'] ?? 0),
                            'limit' => (int)($r['limit'] ?? 20),
                            'truncated' => !empty($r['truncated']),
                            'preview_markdown' => $r['preview_markdown'] ?? '',
                        ];
                    }
                }
                $tools_used[] = $usageEntry;

                // Detect tool ping-pong loops (same tool+args called repeatedly)
                $callSignature = $tool_name . ':' . json_encode($arguments);
                $tool_call_history[] = $callSignature;

                // Check for loops: same signature appearing 3+ times in last 5 calls
                if (count($tool_call_history) >= 5) {
                    $recentCalls = array_slice($tool_call_history, -5);
                    $signatureCounts = array_count_values($recentCalls);
                    if (max($signatureCounts) >= 3) {
                        return $this->agentError(
                            "TOOL_LOOP_DETECTED",
                            "Agent is repeatedly calling the same tool. Breaking loop to prevent infinite execution."
                        );
                    }
                }

                // Increment tool counter and check limit
                $tools_called++;
                if ($tools_called >= $max_tools) {
                    return $this->agentError(
                        "MAX_TOOLS_EXCEEDED",
                        "Agent called {$tools_called} tools (limit: {$max_tools})"
                    );
                }

                // Check timeout
                if (time() - $start_time > $timeout) {
                    return $this->agentError(
                        "TIMEOUT",
                        "Agent exceeded {$timeout} second time limit"
                    );
                }

                // ✅ Inject tool result as USER context (standard practice)
                $messages[] = [
                    'role' => 'user',
                    'content' =>
                        "TOOL RESULT [{$tool_name}]:\n" .
                        json_encode($execution['result'], JSON_PRETTY_PRINT)
                ];

                $params['messages'] = $messages;
                continue;
            } elseif (isset($decoded['final_answer'])) {
                return [
                    'role' => 'assistant',
                    'content' => $decoded['final_answer'],
                    'tools_used' => $tools_used // Include tool metadata for UI
                ];
            } else {
                // No tool_call or final_answer - treat entire response as final answer
                $this->emDebug("No tool_call or final_answer field, using raw content");
                return [
                    'role' => 'assistant',
                    'content' => $content,
                    'tools_used' => $tools_used
                ];
            }
        }

        return $this->agentError(
            "AGENT_MAX_STEPS_EXCEEDED",
            "Agent exceeded maximum allowed steps ({$max_steps})"
        );
    }

    private function callLLMOnce(string $model, array $params, ?int $project_id): array
    {
        $this->initSecureChatAI();

        if (!isset($this->modelConfig[$model])) {
            throw new Exception('Unsupported model: ' . $model);
        }

        $modelConfig = $this->modelConfig[$model];
        $api_endpoint = $modelConfig['api_url'];

        foreach ($modelConfig['required'] as $param) {
            if (empty($params[$param])) {
                throw new Exception('Missing required parameter: ' . $param);
            }
        }

        $filteredParams = $this->filterDefaultParamsForModel($model, $params);

        [$paramName, $dynamicMax, $promptTokens] = $this->computeDynamicMaxTokens($model, $fullPrompt ?? $filteredParams['messages'][0]['content'] ?? '');
        $filteredParams[$paramName] = (int)$dynamicMax;
        $this->emDebug("Dynamic tokens: prompt={$promptTokens}, max={$dynamicMax} for {$model}");

        // Route to appropriate adapter based on model alias
        if (in_array($model, ['ada-002', 'text-embedding-3-small', 'text-embedding-3-large']) || str_contains($model, 'embed')) {
            // Embeddings
            $gpt = new GPTModelRequest($this, $modelConfig, $this->defaultParams, $model);
            $responseData = $gpt->sendRequest($api_endpoint, $filteredParams);

        } elseif (str_contains($model, 'whisper')) {
            // Speech-to-text
            $whisper = new WhisperModelRequest($this, $modelConfig, $this->defaultParams, $model);
            $responseData = $whisper->sendRequest($api_endpoint, $params);

        } elseif (str_contains($model, 'tts')) {
            // Text-to-speech
            $tts = new GPT4oMiniTTSModelRequest($this, $modelConfig, $this->defaultParams, $model);
            $responseData = $tts->sendRequest($api_endpoint, $params);

        } elseif (str_starts_with($model, 'claude') || str_contains($model, 'anthropic')) {
            // Claude (Bedrock format)
            $claude = new ClaudeModelRequest($this, $modelConfig, $this->defaultParams, $model);
            $responseData = $claude->sendRequest($api_endpoint, $filteredParams);

        } else {
            // OpenAI-compatible chat models (Azure AI Foundry, Vertex AI, Llama, etc.)
            $filteredParams = $this->filterDefaultParamsForModel($model, $params);
            $generic = new GenericModelRequest($this, $modelConfig, [], $model);
            $responseData = $generic->sendRequest($api_endpoint, $filteredParams);
            // PHI-safe: log only response metadata (token usage, model, shape) — never the
            // raw API response body, which can contain response PHI.
            $this->emDebug("RAW GenericModelRequest API RESPONSE (metadata only)", $this->summarizeResponseForDebug($responseData));
        }

        $normalizedResponse = $this->normalizeResponse($responseData, $model);

        if ($project_id) {
            $this->logInteraction($project_id, $params, $responseData);
        }

        return $normalizedResponse;
    }

    private function executeToolCall(
        string $tool_name,
        array $arguments,
        array $tools,
        ?int $project_id,
        string $model = 'o3-mini',
        ?string $username = null
    ): array {
        // Built-in: spawnAgent (before pipeline lookup)
        if ($tool_name === 'spawnAgent') {
            return $this->handleSpawnAgentCall($arguments, $project_id, $model, $username);
        }

        // Build tool use + context
        $toolUse = new ToolUse($tool_name, $arguments);
        $context = new ToolContext($project_id, $username);
        $context->set('secure_chat_ai_instance', $this);
        $context->set('guzzle_client', $this->guzzleClient);
        $context->set('model', $model);

        // Phase 1: Lookup tool config from JSON registry
        $toolConfig = null;
        foreach ($tools as $t) {
            if (($t['name'] ?? null) === $tool_name) {
                $toolConfig = $t;
                break;
            }
        }

        if (!$toolConfig) {
            return $this->agentError("UNKNOWN_TOOL", "Tool '{$tool_name}' not registered");
        }

        // Wrap config as typed tool
        $toolAdapter = new JsonConfigToolAdapter($toolConfig);

        // Build pipeline with hooks from config (system + project)
        $pipeline = new ToolPipeline(
            resolver: fn() => $toolAdapter,
            preHooks: $this->loadPreToolUseHooks($project_id),
            postHooks: $this->loadPostToolUseHooks($project_id)
        );

        // Run pipeline (Phases 2-7: parse, validate, pre-hooks, permissions, execute, post-hooks)
        $result = $pipeline->handle($toolUse, $context);

        // Preserve MISSING_PARAMETERS behavior: return error=false so agent can ask naturally
        if ($result->isError && str_contains($result->errorMessage ?? '', 'Missing required')) {
            $missing = str_replace('Missing required parameters: ', '', $result->errorMessage);
            return [
                'error'   => false,
                'type'    => 'MISSING_PARAMETERS',
                'missing' => array_map('trim', explode(',', $missing)),
                'result'  => [
                    'status'         => 'incomplete',
                    'message'        => $result->errorMessage,
                    'missing_fields' => array_map('trim', explode(',', $missing))
                ]
            ];
        }

        // Hook denials should be fed back to the agent (not treated as fatal errors)
        // so the agent can explain what happened to the user
        if ($result->isError && (
            str_contains($result->errorMessage ?? '', 'denied by hook') ||
            str_contains($result->errorMessage ?? '', 'Blocked by')
        )) {
            return [
                'error'  => false,
                'result' => [
                    'status'  => 'denied',
                    'message' => $result->errorMessage,
                ]
            ];
        }

        return $result->toArray();
    }

    /**
     * Load PreToolUse hooks from system + project settings.
     *
     * @return PreToolUseHook[]
     */
    private function loadPreToolUseHooks(?int $projectId = null): array
    {
        // ProjectAccessPreHook is always-on — core governance, not configurable
        $builtin = [new ProjectAccessPreHook()];
        $system  = $this->loadHooksFromSetting('pre_tool_use_hooks', PreToolUseHook::class, null);
        $project = $projectId
            ? $this->loadHooksFromSetting('project_pre_tool_use_hooks', PreToolUseHook::class, $projectId)
            : [];
        $hooks = array_merge($builtin, $system, $project);

        // Project-level opt-out: the PHI field protection hook is ON by default,
        // but a project may disable it (all current models accept PHI).
        if ($projectId && $this->getProjectSetting('disable_phi_field_hook', $projectId)) {
            $hooks = array_values(array_filter($hooks, fn($hook) => !$this->isPhiFieldHook($hook)));
            $this->emDebug("PHI field pre-hook disabled via project setting for project {$projectId}");
        }

        return $hooks;
    }

    /**
     * True if the given hook instance is the PHI field protection hook
     * (matched by class basename so it works regardless of installed EM version).
     */
    private function isPhiFieldHook($hook): bool
    {
        $class = get_class($hook);
        $needle = '\\PhiFieldPreHook';
        return $class === 'Stanford\\REDCapAgentRecordTools\\PhiFieldPreHook'
            || substr($class, -strlen($needle)) === $needle;
    }

    /**
     * Load PostToolUse hooks from system + project settings.
     *
     * @return PostToolUseHook[]
     */
    private function loadPostToolUseHooks(?int $projectId = null): array
    {
        $system  = $this->loadHooksFromSetting('post_tool_use_hooks', PostToolUseHook::class, null);
        $project = $projectId
            ? $this->loadHooksFromSetting('project_post_tool_use_hooks', PostToolUseHook::class, $projectId)
            : [];
        return array_merge($system, $project);
    }

    /**
     * Load and instantiate hook classes from a comma-separated setting.
     *
     * @param string $settingKey  Setting name
     * @param string $interfaceName  Required interface
     * @param int|null $projectId  If provided, reads project setting; otherwise system setting
     * @return array
     */
    private function loadHooksFromSetting(string $settingKey, string $interfaceName, ?int $projectId = null): array
    {
        $raw = $projectId !== null
            ? $this->getProjectSetting($settingKey, $projectId)
            : $this->getSystemSetting($settingKey);

        if (empty($raw)) {
            return [];
        }

        $classNames = array_map('trim', explode(',', $raw));
        $hooks = [];

        foreach ($classNames as $className) {
            if (empty($className) || !class_exists($className)) {
                $this->emDebug("Hook class not found: {$className}");
                continue;
            }

            $instance = new $className();
            if ($instance instanceof $interfaceName) {
                $hooks[] = $instance;
            } else {
                $this->emDebug("Hook class {$className} does not implement {$interfaceName}");
            }
        }

        return $hooks;
    }

    private function askForMoreInfoText(string $tool_name, array $missing_fields): string
    {
        return "I need: " . implode(", ", $missing_fields) . " to use {$tool_name}.";
    }

    private function agentError(string $type, string $message): array
    {
        return [
            'error' => true,
            'type' => $type,
            'message' => $message
        ];
    }

    /**
     * Handle the built-in spawnAgent tool call.
     */
    private function handleSpawnAgentCall(array $arguments, ?int $project_id, string $model, ?string $username = null): array
    {
        $prompt = $arguments['prompt'] ?? '';
        if (empty($prompt)) {
            return $this->agentError("MISSING_PROMPT", "spawnAgent requires a 'prompt' argument");
        }

        $maxDepth = (int) ($this->getSystemSetting('agent_max_subagent_depth') ?? 1);
        if ($maxDepth <= 0) {
            return $this->agentError("SUBAGENTS_DISABLED", "Sub-agents are disabled (agent_max_subagent_depth=0)");
        }

        $allowedTools = $arguments['allowed_tools'] ?? [];

        return $this->spawnAgent(
            prompt: $prompt,
            model: $model,
            projectId: $project_id,
            username: $username,
            depth: 1,
            maxDepth: $maxDepth,
            allowedTools: $allowedTools
        );
    }

    /**
     * Spawn a sub-agent to handle a task independently.
     *
     * The sub-agent runs a fresh agent loop with reduced limits
     * and returns its final answer as a tool result.
     */
    private function spawnAgent(
        string $prompt,
        string $model,
        ?int $projectId,
        ?string $username = null,
        int $depth = 1,
        int $maxDepth = 1,
        array $allowedTools = []
    ): array
    {
        $this->emDebug("SPAWN AGENT (depth={$depth}/{$maxDepth})", [
            'prompt_preview' => substr($prompt, 0, 100),
            'model' => $model,
            'allowed_tools' => $allowedTools ?: '(all)'
        ]);

        // Depth guard
        if ($depth > $maxDepth) {
            return $this->agentError(
                "MAX_DEPTH_EXCEEDED",
                "Sub-agent depth {$depth} exceeds max {$maxDepth}"
            );
        }

        // Load tools for the project
        $allTools = $this->loadToolsForProject($projectId);

        // Scope tools if requested
        if (!empty($allowedTools)) {
            $allTools = array_values(array_filter($allTools, function ($t) use ($allowedTools) {
                return in_array($t['name'] ?? '', $allowedTools);
            }));
        }

        // Build fresh messages from the prompt
        $messages = [
            ['role' => 'user', 'content' => $prompt]
        ];

        // Reduced limits for sub-agents
        $params = [
            'messages' => $messages,
            'max_tokens' => 4000,
        ];

        // Run child agent loop
        $result = $this->runAgentLoop(
            model: $model,
            params: $params,
            project_id: $projectId,
            username: $username,
            depth: $depth,
            maxDepth: $maxDepth
        );

        // Format result for parent
        if (!empty($result['error'])) {
            return [
                'error'  => false, // Don't propagate as hard error to parent
                'result' => [
                    'status' => 'failed',
                    'error'  => $result['message'] ?? 'Sub-agent failed',
                ]
            ];
        }

        $finalAnswer = $result['content'] ?? $result['message'] ?? '(no result)';

        return [
            'error'  => false,
            'result' => [
                'status' => 'completed',
                'result' => $finalAnswer,
            ]
        ];
    }

    /**
     * Cap tool result size to prevent token overflow
     * Intelligently truncates while preserving structure and adding metadata
     */
    private function capToolResultSize($result, int $maxChars)
    {
        $jsonResult = json_encode($result, JSON_UNESCAPED_UNICODE);
        $originalSize = strlen($jsonResult);

        // If under limit, return as-is
        if ($originalSize <= $maxChars) {
            return $result;
        }

        $truncated = $result;
        $wasTruncated = false;

        // Handle arrays - truncate items
        if (is_array($result) && array_is_list($result)) {
            $itemCount = count($result);
            $kept = [];
            $currentSize = 2; // [] brackets

            foreach ($result as $item) {
                $itemJson = json_encode($item, JSON_UNESCAPED_UNICODE);
                $itemSize = strlen($itemJson) + 1; // +1 for comma

                if ($currentSize + $itemSize > $maxChars - 200) { // Reserve 200 chars for metadata
                    $wasTruncated = true;
                    break;
                }

                $kept[] = $item;
                $currentSize += $itemSize;
            }

            $truncated = $kept;
            if ($wasTruncated) {
                $truncated[] = [
                    '_truncated' => true,
                    '_original_count' => $itemCount,
                    '_returned_count' => count($kept),
                    '_message' => "Result truncated to fit token budget. Showing " . count($kept) . " of $itemCount items."
                ];
            }
        }
        // Handle objects - recursively cap nested values
        elseif (is_array($result)) {
            $currentSize = 2; // {} brackets
            $truncated = [];

            foreach ($result as $key => $value) {
                $valueJson = json_encode($value, JSON_UNESCAPED_UNICODE);
                $itemSize = strlen($key) + strlen($valueJson) + 4; // key + value + quotes/colon

                if ($currentSize + $itemSize > $maxChars - 200) {
                    $wasTruncated = true;
                    break;
                }

                $truncated[$key] = $value;
                $currentSize += $itemSize;
            }

            if ($wasTruncated) {
                $truncated['_truncated'] = true;
                $truncated['_message'] = "Result truncated to fit token budget.";
            }
        }
        // Handle strings - simple truncation
        elseif (is_string($result)) {
            $truncated = substr($result, 0, $maxChars - 100);
            $truncated .= "\n\n[... truncated " . ($originalSize - strlen($truncated)) . " characters to fit token budget]";
        }

        return $truncated;
    }

    /**
     * Final cleanup: ensure response is always clean, user-ready text
     * - Extract text from stringified JSON (agent schema or accidental)
     * - Convert error objects to polite messages
     * - Never return raw JSON strings or error arrays to the UI
     */
    private function sanitizeOutputForUI(array $response): array
    {
        // Embedding responses (ada-002) should pass through unchanged
        // They have 'data' field with embedding vectors, not 'content'
        if (isset($response['data']) && !isset($response['content'])) {
            $this->emDebug("Passing through embedding response in sanitizeOutputForUI");
            return $response;
        }

        // TTS responses should pass through unchanged (have audio_base64)
        if (isset($response['audio_base64'])) {
            $this->emDebug("Passing through TTS response in sanitizeOutputForUI");
            return $response;
        }

        // STT/Whisper responses should pass through unchanged (have 'text' from transcription)
        // Raw whisper responses don't have 'role' field
        if (isset($response['text']) && !isset($response['role'])) {
            $this->emDebug("Passing through STT response in sanitizeOutputForUI");
            return $response;
        }

        // Handle error responses with friendly messages
        if (!empty($response['error'])) {
            $type = $response['type'] ?? 'UNKNOWN_ERROR';
            $msg = $response['message'] ?? 'An error occurred';

            $friendlyMessages = [
                'TIMEOUT' => "I apologize, but that request took longer than expected. Please try again in a moment.",
                'MAX_TOOLS_EXCEEDED' => "I apologize, but I needed to use too many tools for that request. Could you try simplifying your question?",
                'AGENT_MAX_STEPS_EXCEEDED' => "I apologize, but I couldn't complete that task efficiently. Could you try breaking it into smaller requests?",
                'TOOL_LOOP_DETECTED' => "I apologize, but I got stuck in a loop trying to complete that request. Could you try rephrasing your question?",
                'UNKNOWN_TOOL' => "I apologize, but I don't have access to that capability right now.",
                'MISCONFIGURED_AGENT_TOOLS' => "I apologize, but I'm experiencing technical difficulties. Please contact your administrator.",
                'REDCAP_API_ERROR' => "I apologize, but I'm having trouble accessing that data right now. Please try again in a moment.",
                'NETWORK_ERROR' => "I apologize, but I'm experiencing network difficulties. Please wait a moment and try again.",
            ];

            $politeMessage = $friendlyMessages[$type] ??
                "I apologize, but I'm experiencing technical difficulties. Please wait a moment and try again.";

            return [
                'role' => 'assistant',
                'content' => $politeMessage
            ];
        }

        $content = trim($response['content'] ?? '');

        // Decode HTML entities (in case of double-encoding)
        $content = html_entity_decode($content, ENT_QUOTES, 'UTF-8');

        // Try to detect and parse JSON responses
        if (!empty($content) && ($content[0] === '{' || str_starts_with($content, '```json'))) {
            // Strip markdown code fences if present
            $content = preg_replace('/^```json\s*|\s*```$/s', '', $content);
            $content = trim($content);

            // Clean control characters that break JSON parsing (same as agent loop)
            $cleanContent = preg_replace('/[\x00-\x1F]/', '', $content);

            $decoded = json_decode($cleanContent, true);
            $parseError = json_last_error();

            if ($parseError === JSON_ERROR_NONE && is_array($decoded)) {
                if (!empty($response['preserve_structure'])) {
                    // Keep the JSON string as-is for structured output
                    $response['content'] = $content;
                    return $response;
                }

                // Extract from agent schema format
                if (isset($decoded['final_answer'])) {
                    $content = $decoded['final_answer'];
                } elseif (isset($decoded['tool_call'])) {
                    // If we're seeing a tool_call in final output, something went wrong
                    $content = "I apologize, but I wasn't able to complete that request properly. Could you try rephrasing?";
                } elseif (isset($decoded['content'])) {
                    // Some models nest content in JSON
                    $content = $decoded['content'];
                } elseif (isset($decoded['message'])) {
                    $content = $decoded['message'];
                }
                // Otherwise leave the JSON string as-is (might be intentional structured output)
            }
        }

        // EMERGENCY BACKSTOP: If content STILL looks like our JSON schema, strip it one more time
        // This handles edge cases where json_decode failed or we didn't extract properly
        // Works even with truncated JSON (missing closing brace/quote)
        if (!empty($content) && preg_match('/\{\s*"final_answer"\s*:\s*"(.*)$/s', $content, $match)) {
            $extracted = $match[1];
            // Remove trailing garbage (incomplete JSON structure)
            $extracted = preg_replace('/["}\s]*$/', '', $extracted);
            // Unescape JSON string escapes
            $content = str_replace(['\n', '\r', '\t', '\"', '\\\\'], ["\n", "\r", "\t", '"', '\\'], $extracted);
        }

        $sanitized = [
            'role' => $response['role'] ?? 'assistant',
            'content' => $content,
            'model' => $response['model'] ?? null,
            'usage' => $response['usage'] ?? null
        ];

        // Preserve tool metadata if present (for UI display)
        if (!empty($response['tools_used'])) {
            $sanitized['tools_used'] = $response['tools_used'];
        }

        return $sanitized;
    }

    /**
     * Convert a messages array to a readable string for token estimation
     * and summarization prompts.
     */
    private function messagesToText(array $messages): string
    {
        $lines = [];
        foreach ($messages as $msg) {
            $role = $msg['role'] ?? 'unknown';
            $content = $msg['content'] ?? '';

            // Handle content arrays (some models use [{type: "text", text: "..."}])
            if (is_array($content)) {
                $parts = [];
                foreach ($content as $block) {
                    if (isset($block['text'])) $parts[] = $block['text'];
                }
                $content = implode("\n", $parts);
            }

            $lines[] = "[{$role}]: {$content}";
        }
        return implode("\n", $lines);
    }

    /**
     * Compact messages if they exceed the model's context window.
     *
     * Opt-in via params:
     *   compact_if_needed  (bool)   Enable compaction
     *   compact_keep_recent (int)   Recent messages to keep (default 6)
     *   compact_model       (string) Cheap model for summarization
     *
     * Returns updated $params with compacted messages.
     */
    private function compactMessagesIfNeeded(array $params, string $model, ?int $contextWindowOverride = null): array
    {
        if (empty($params['compact_if_needed'])) {
            return $params;
        }

        $messages = $params['messages'] ?? [];
        if (empty($messages)) {
            return $params;
        }

        // Estimate current token count
        $messagesText = $this->messagesToText($messages);
        $currentTokens = $this->estimateTokens($messagesText, $model);

        // Get context window for this model (allow override for testing)
        $contextWindow = $contextWindowOverride ?? ($this->getModelContextSpec($model)['context'] ?? 128000);
        $threshold = (int) ($contextWindow * 0.8);

        if ($currentTokens < $threshold) {
            $this->emDebug("Compaction: skipping ({$currentTokens} tokens < {$threshold} threshold)");
            return $params;
        }

        $keepRecent = (int) ($params['compact_keep_recent'] ?? 6);
        $summarizeModel = $params['compact_model'] ?? $model;

        // Don't compact if too few messages
        if (count($messages) <= $keepRecent + 2) {
            return $params;
        }

        $this->emDebug("Compaction: triggering ({$currentTokens} tokens > {$threshold} threshold)", [
            'total_messages' => count($messages),
            'keep_recent' => $keepRecent,
            'summarize_model' => $summarizeModel,
        ]);

        // Split into old + recent
        $recentMessages = array_slice($messages, -$keepRecent);
        $oldMessages = array_slice($messages, 0, count($messages) - $keepRecent);

        // Separate system messages from conversation messages
        $systemMessages = [];
        $conversationMessages = [];
        foreach ($oldMessages as $msg) {
            if (($msg['role'] ?? '') === 'system') {
                $systemMessages[] = $msg;
            } else {
                $conversationMessages[] = $msg;
            }
        }

        if (empty($conversationMessages)) {
            return $params;
        }

        // Summarize old conversation messages
        $oldText = $this->messagesToText($conversationMessages);
        $summaryPrompt = "Concisely summarize this conversation in 3-5 sentences. "
            . "Preserve key facts, decisions, and context. Be brief but comprehensive.\n\n"
            . $oldText;

        try {
            // Use correct token param for this model
            $o1Models = ['o1', 'o3-mini', 'o3', 'o4-mini'];
            $tokenParam = in_array($summarizeModel, $o1Models)
                ? 'max_completion_tokens'
                : 'max_tokens';

            $summaryResponse = $this->callLLMOnce($summarizeModel, [
                'messages' => [
                    ['role' => 'user', 'content' => $summaryPrompt]
                ],
                $tokenParam => 500,
            ], null);

            $summary = trim($summaryResponse['content'] ?? '');

            if (empty($summary)) {
                $this->emDebug("Compaction: summarization returned empty, skipping");
                return $params;
            }

            // Rebuild messages: system + summary + recent
            $summaryMsg = [
                'role' => 'system',
                'content' => "Previous conversation summary: {$summary}"
            ];

            $compactedMessages = array_merge(
                $systemMessages,
                [$summaryMsg],
                $recentMessages
            );

            // Log metrics
            $compactedText = $this->messagesToText($compactedMessages);
            $compactedTokens = $this->estimateTokens($compactedText, $model);
            $this->emDebug("Compaction: complete", [
                'before_tokens' => $currentTokens,
                'after_tokens' => $compactedTokens,
                'reduction' => round((1 - $compactedTokens / $currentTokens) * 100, 1) . '%',
                'before_messages' => count($messages),
                'after_messages' => count($compactedMessages),
            ]);

            $params['messages'] = $compactedMessages;
            return $params;

        } catch (\Throwable $e) {
            $this->emDebug("Compaction: summarization failed, proceeding without: " . $e->getMessage());
            return $params;
        }
    }

    /**
     * Get context window spec for a model.
     */
    private function getModelContextSpec(string $model): array
    {
        $specs = [
            'o1'               => ['context' => 200000],
            'o3'               => ['context' => 200000],
            'o3-mini'          => ['context' => 200000],
            'o4-mini'          => ['context' => 200000],
            'gpt-4o'           => ['context' => 128000],
            'gpt-4-1'          => ['context' => 1000000],
            'gpt-4-1-nano'     => ['context' => 1000000],
            'gpt-5'            => ['context' => 400000],
            'gpt-5-4'          => ['context' => 400000],
            'gpt-5-4-nano'     => ['context' => 400000],
            'gpt-5-nano'       => ['context' => 400000],
            'grok-3'           => ['context' => 128000],
            'grok-3-mini'      => ['context' => 128000],
            'deepseek'         => ['context' => 128000],
            'deepseek-r1'      => ['context' => 128000],
            'claude'           => ['context' => 200000],
            'claude-sonnet-3.5' => ['context' => 200000],
            'claude-sonnet-3.7' => ['context' => 200000],
            'claude-haiku'     => ['context' => 200000],
            'claude-haiku-4.5' => ['context' => 200000],
            'claude-opus-4'    => ['context' => 200000],
            'claude-opus-4-1'  => ['context' => 200000],
            'claude-opus-4-6'  => ['context' => 200000],
            'claude-opus-4-7'  => ['context' => 200000],
            'claude-sonnet-4'  => ['context' => 200000],
            'claude-sonnet-4-5' => ['context' => 200000],
            'claude-sonnet-4-6' => ['context' => 200000],
            'gemini-2.5-pro'   => ['context' => 1000000],
            'gemini-2.5-flash' => ['context' => 1000000],
            'llama3370b'       => ['context' => 128000],
            'llama-Maverick'   => ['context' => 128000],
            'llama-4-scout'    => ['context' => 128000],
        ];
        return $specs[$model] ?? ['context' => 128000];
    }

    private function estimateTokens(string $text, string $model): int {
        try {
            $this->encoderProvider ??= new EncoderProvider();
            $encoder = $this->encoderProvider->getForModel($model);
            return count($encoder->encode($text));
        } catch (\Throwable $e) {
            // Fallback for models not in tiktoken registry (e.g. llama-Maverick)
            return (int)(mb_strlen($text) / 4);
        }
    }

    private function computeDynamicMaxTokens(string $model, string $prompt): array {
        $modelSpecs = [
            'o1' => [
                'context' => 200000,
                'output_max' => 100000,
                'param' => 'max_completion_tokens',
                'buffer' => 25000
            ],
            'o3' => [
                'context' => 200000,
                'output_max' => 100000,
                'param' => 'max_completion_tokens',
                'buffer' => 25000
            ],
            'o3-mini' => [
                'context' => 200000,
                'output_max' => 100000,
                'param' => 'max_completion_tokens',
                'buffer' => 25000
            ],
            'o4-mini' => [
                'context' => 200000,
                'output_max' => 100000,
                'param' => 'max_completion_tokens',
                'buffer' => 25000
            ],
            'gpt-4-1' => [
                'context' => 1000000,
                'output_max' => 128000,
                'param' => 'max_tokens',
                'buffer' => 2000
            ],
            'gpt-4-1-nano' => [
                'context' => 1000000,
                'output_max' => 128000,
                'param' => 'max_tokens',
                'buffer' => 2000
            ],
            'gpt-5' => [
                'context' => 400000,
                'output_max' => 128000,
                'param' => 'max_tokens',
                'buffer' => 2000
            ],
            'gpt-5-4' => [
                'context' => 400000,
                'output_max' => 128000,
                'param' => 'max_tokens',
                'buffer' => 2000
            ],
            'gpt-5-4-nano' => [
                'context' => 400000,
                'output_max' => 128000,
                'param' => 'max_tokens',
                'buffer' => 2000
            ],
            'claude' => [
                'context' => 200000,
                'output_max' => 8192,
                'param' => 'max_tokens',
                'buffer' => 2000
            ],
            'claude-sonnet-3.5' => [
                'context' => 200000,
                'output_max' => 8192,
                'param' => 'max_tokens',
                'buffer' => 2000
            ],
            'claude-sonnet-3.7' => [
                'context' => 200000,
                'output_max' => 8192,
                'param' => 'max_tokens',
                'buffer' => 2000
            ],
            'claude-haiku-4.5' => [
                'context' => 200000,
                'output_max' => 8192,
                'param' => 'max_tokens',
                'buffer' => 2000
            ],
            'claude-opus-4' => [
                'context' => 200000,
                'output_max' => 32000,
                'param' => 'max_tokens',
                'buffer' => 2000
            ],
            'claude-sonnet-4' => [
                'context' => 200000,
                'output_max' => 16000,
                'param' => 'max_tokens',
                'buffer' => 2000
            ]
        ];
        $spec = $modelSpecs[$model] ?? null;
        if (!$spec) return ['max_tokens', 16384, 0];

        $promptTokens = $this->estimateTokens($prompt, $model);
        $available = $spec['context'] - $promptTokens - $spec['buffer'];
        $final = min($available, $spec['output_max']);
        $final = max(1024, $final);  // Min fallback

        // Metadata only (computed token ceiling) — no prompt/response content. Routed
        // through emDebug so it respects debug-mode gating instead of always-on error_log.
        $this->emDebug("computeDynamicMaxTokens", [
            'model'         => $model,
            'prompt_tokens' => $promptTokens,
            'final_max'     => $final,
        ]);

        return [$spec['param'], $final, $promptTokens];
    }


// OPTIONAL FOR LATER , MODEL SPECIFIC STUFF
private function modelSupportsNativeTools(string $model): bool
{
    // Intentionally disabled for initial agent rollout
    return false;

    /*
    $raw = (string) ($this->getSystemSetting('agent_tools_native_models') ?? '');
    $allow = array_filter(array_map('trim', explode(',', $raw)));
    if (empty($allow)) return false;

    return in_array($model, $allow, true);
    */
}

private function toOpenAIToolsShape(array $tools): array
{
    // Intentionally disabled for initial agent rollout
    return [];

    /*
    $out = [];
    foreach ($tools as $t) {
        if (empty($t['name'])) continue;
        $out[] = [
            'type' => 'function',
            'function' => [
                'name' => $t['name'],
                'description' => $t['description'] ?? '',
                'parameters' => $t['parameters'] ?? [
                    'type' => 'object',
                    'properties' => []
                ]
            ]
        ];
    }
    return $out;
    */
}
// OPTIONAL FOR LATER , MODEL SPECIFIC STUFF


    /**
     * Build a PHI-safe metadata summary of an LLM request/response for debug logging.
     *
     * Prompt and response BODIES may contain PHI, so they must never be written to the
     * debug logs. This helper extracts only non-content metadata (token usage, model,
     * role, finish reason, structural shape and content length) that is safe to log.
     *
     * @param mixed $response Raw or normalized response from an LLM adapter.
     * @return array Safe-to-log metadata describing the response (no content body).
     */
    private function summarizeResponseForDebug($response): array
    {
        // Non-array payloads carry no structured content to redact; report the type only.
        if (!is_array($response)) {
            return ['type' => gettype($response)];
        }

        // Resolve the message content from either the normalized shape ('content')
        // or the raw OpenAI-compatible shape ('choices[0].message.content') WITHOUT
        // ever including the text itself in the returned metadata.
        $content = $response['content']
            ?? ($response['choices'][0]['message']['content'] ?? null);

        return [
            // Key names only (e.g. "content", "usage") — never their values.
            'top_level_keys' => array_keys($response),
            'model'          => $response['model'] ?? null,
            'role'           => $response['role']
                ?? ($response['choices'][0]['message']['role'] ?? null),
            'finish_reason'  => $response['choices'][0]['finish_reason'] ?? null,
            // Token counts are metadata, not content — safe to log.
            'usage'          => $response['usage'] ?? null,
            'choices_count'  => isset($response['choices']) && is_array($response['choices'])
                ? count($response['choices'])
                : null,
            // Length and shape signals only — never the text itself.
            'content_length' => is_string($content) ? strlen($content) : null,
            'has_tool_call'  => is_string($content) && str_contains($content, 'tool_call'),
        ];
    }

    private function normalizeResponse($response, $model)
    {
        $normalized = [];

        $this->emDebug("normalizeResponse called", [
            'model' => $model,
            'has_data' => isset($response['data']),
            'has_choices' => isset($response['choices'])
        ]);

        // Embedding models return their own format - pass through unchanged
        if (str_contains($model, 'embed') || $model === 'ada-002') {
            $this->emDebug("Passing through embedding response unchanged for model: {$model}");
            return $response;
        }

        if (str_starts_with($model, 'claude') || str_contains($model, 'anthropic')) {
            $normalized['content'] = $response['content'][0]['text'] ?? '';
            $normalized['role'] = $response['role'] ?? 'assistant';
            $normalized['model'] = $response['model'] ?? 'claude';
            $normalized['usage'] = [
                'prompt_tokens' => $response['usage']['input_tokens'] ?? 0,
                'completion_tokens' => $response['usage']['output_tokens'] ?? 0,
                'total_tokens' => ($response['usage']['input_tokens'] ?? 0) + ($response['usage']['output_tokens'] ?? 0)
            ];
        } elseif (str_contains($model, 'tts')) {
            $normalized['audio_base64'] = $response['audio_base64'] ?? '';
            $normalized['content_type'] = $response['content_type'] ?? 'audio/mpeg';
            $normalized['model'] = $model;
        } elseif (isset($response['choices'][0]['message']['content'])) {
            // OpenAI-compatible format (catches all chat models dynamically)
            $normalized['content'] = $response['choices'][0]['message']['content'] ?? '';
            $decoded = json_decode($normalized['content'], true);
            if (json_last_error() === JSON_ERROR_NONE && is_array($decoded)) {
                $normalized['structured_output'] = $decoded;
                // Only mark preserve_structure when the JSON looks like a genuine
                // tool-call / structured-output payload, NOT when it's an agent-mode
                // response envelope ({final_answer|tool_call|thinking}). The agent
                // schema is consumed by runAgentLoop before sanitizeOutputForUI,
                // so by the time we get here it must already be unwrapped — any
                // remaining JSON is either an unwrapped envelope (bug) or
                // legitimate structured output. Inspect the shape.
                $isAgentEnvelope = isset($decoded['final_answer'])
                    || isset($decoded['tool_call'])
                    || isset($decoded['thinking']);
                if (!$isAgentEnvelope) {
                    $normalized['preserve_structure'] = true;
                }
            }
            $normalized['role'] = $response['choices'][0]['message']['role'] ?? 'assistant';
            $normalized['model'] = $response['model'] ?? $model;
            $normalized['usage'] = [
                'prompt_tokens' => $response['usage']['prompt_tokens'] ?? 0,
                'completion_tokens' => $response['usage']['completion_tokens'] ?? 0,
                'total_tokens' => $response['usage']['total_tokens'] ?? 0
            ];
        } else {
            $normalized = $response;
        }

        // $this->emDebug("normalized responseData", $normalized);
        return $normalized;
    }

    /**
     * Build a rehydrated session from dedicated one-row-per-turn records.
     * Each row already holds the real user prompt and final assistant answer,
     * so this is a straight map — no filtering of agent-loop bookkeeping, and
     * errored turns keep their user prompt (they are NOT skipped).
     */
    private function buildSessionFromTurns($session_id, $pid, array $turnLogs)
    {
        $messages = [];
        $models = [];
        $total_tokens = 0;
        $first_timestamp = null;
        $last_timestamp = null;

        foreach ($turnLogs as $index => $log) {
            if (!empty($log['user_message'])) {
                $messages[] = [
                    'role' => 'user',
                    'content' => $log['user_message'],
                    'turn' => $index + 1,
                ];
            }
            if (!empty($log['assistant_response'])) {
                $messages[] = [
                    'role' => 'assistant',
                    'content' => $log['assistant_response'],
                    'turn' => $index + 1,
                    'tools_used' => $log['tools_used'] ?? null,
                ];
            }
            if (!empty($log['model'])) $models[$log['model']] = true;
            if (!empty($log['usage']['total_tokens'])) $total_tokens += $log['usage']['total_tokens'];
            if ($first_timestamp === null) $first_timestamp = $log['timestamp'];
            $last_timestamp = $log['timestamp'];
        }

        return [
            'session_id' => $session_id,
            'messages' => $messages,
            'metadata' => [
                'project_id' => $pid,
                'start_time' => $first_timestamp,
                'end_time' => $last_timestamp,
                'duration_seconds' => $first_timestamp && $last_timestamp ?
                    strtotime($last_timestamp) - strtotime($first_timestamp) : 0
            ],
            'stats' => [
                'total_turns' => count($turnLogs),
                'total_tokens' => $total_tokens,
                'models_used' => array_keys($models)
            ]
        ];
    }

    /**
     * Log ONE conversation turn (real user prompt + final assistant answer) as
     * a dedicated record, distinct from the per-LLM-call audit rows written by
     * logInteraction(). rehydrateProjectSession reads THESE rows first to
     * rebuild chat history reliably.
     *
     * Two-store design (intentional, not redundant to remove): logInteraction()
     * still writes its per-LLM-call rows — those remain the AUDIT trail (every
     * model call, consumed by getSecureChatLogsBySession and admin tooling).
     * CappyChatTurn rows are the REHYDRATION source (one clean row per turn).
     * They overlap in content but serve different consumers; do not drop the
     * per-call rows on the assumption turn rows supersede them.
     *
     * Skipped when there is no session_id, or when the caller opts out via
     * $requestData['log_turn'] === false (e.g. the client-side compaction
     * summary call, which reuses the session id but is not a real user turn).
     */
    private function logConversationTurn($project_id, $requestData, $responseData, $username = null)
    {
        if (empty($requestData['session_id'])) {
            return;
        }
        if (array_key_exists('log_turn', $requestData) && $requestData['log_turn'] === false) {
            return;
        }

        // Real user prompt = last user message in the INCOMING messages. The
        // agent loop's injected TOOL RESULT messages live on a local copy inside
        // runAgentLoop, so $requestData['messages'] here is still the clean UI
        // context.
        $userMessage = '';
        if (!empty($requestData['messages']) && is_array($requestData['messages'])) {
            foreach (array_reverse($requestData['messages']) as $msg) {
                if (($msg['role'] ?? '') === 'user') {
                    $userMessage = (string)($msg['content'] ?? '');
                    break;
                }
            }
        }

        $isError = !empty($responseData['error']);
        $assistant = $responseData['content']
            ?? ($responseData['choices'][0]['message']['content'] ?? '');

        // Skip empty turns — a row with neither a user prompt nor an assistant
        // answer (e.g. an embedding/structured response that happens to carry a
        // session_id) is junk that rehydrates to a blank bubble.
        if (trim((string)$userMessage) === '' && trim((string)$assistant) === '') {
            return;
        }

        $atomicPayload = [
            'project_id'         => $project_id,
            'session_id'         => $requestData['session_id'],
            // Owner of the turn. rehydrateProjectSession filters by this so one
            // user can't rebuild another user's chat from a guessed session_id.
            'username'           => $username !== null ? (string)$username : null,
            'model'              => $responseData['model'] ?? ($requestData['model'] ?? 'unknown'),
            'timestamp'          => date('Y-m-d H:i:s'),
            'user_message'       => $userMessage,
            'assistant_response' => $assistant,
        ];
        if (!empty($responseData['tools_used'])) {
            $atomicPayload['tools_used'] = $responseData['tools_used'];
        }
        if (!empty($responseData['usage'])) {
            $atomicPayload['usage'] = $responseData['usage'];
        }
        if ($isError) {
            $atomicPayload['error'] = true;
        }

        $action = new SecureChatLog($this);
        $action->setValue('message', json_encode($atomicPayload));
        $action->setValue('record', $isError ? 'CappyChatTurnError' : 'CappyChatTurn');
        if ($project_id) {
            $action->setValue('project_id', $project_id);
        }
        $action->setValue('session_id', $requestData['session_id']);
        if (!empty($atomicPayload['model'])) {
            $action->setValue('model', $atomicPayload['model']);
        }
        $action->save();
    }

    private function logInteraction($project_id, $requestData, $responseData)
    {
        // Extract atomic log data - only the current turn, not full conversation history
        $atomicPayload = [
            'project_id' => $project_id,
            'session_id' => $requestData['session_id'] ?? null,
            'model' => $responseData['model'] ?? ($requestData['model'] ?? 'unknown'),
            'timestamp' => date('Y-m-d H:i:s')
        ];

        // Extract the last user message from the messages array
        if (!empty($requestData['messages']) && is_array($requestData['messages'])) {
            // Find the last user message (skip system messages)
            $userMessages = array_filter($requestData['messages'], function($msg) {
                return ($msg['role'] ?? '') === 'user';
            });
            $lastUserMessage = !empty($userMessages) ? array_values($userMessages)[count($userMessages) - 1] : null;

            if ($lastUserMessage) {
                $atomicPayload['user_message'] = $lastUserMessage['content'] ?? '';
            }
        }

        // Extract assistant response
        if (!empty($responseData)) {
            // Handle normalized response structure
            if (isset($responseData['content'])) {
                $atomicPayload['assistant_response'] = $responseData['content'];
            } elseif (isset($responseData['choices'][0]['message']['content'])) {
                $atomicPayload['assistant_response'] = $responseData['choices'][0]['message']['content'];
            }

            // Extract usage data
            if (isset($responseData['usage'])) {
                $atomicPayload['usage'] = $responseData['usage'];
            }

            // Extract any tool usage metadata
            if (isset($responseData['tools_used'])) {
                $atomicPayload['tools_used'] = $responseData['tools_used'];
            }
        }

        $action = new SecureChatLog($this);
        $action->setValue('message', json_encode($atomicPayload));
        $action->setValue('record', 'SecureChatLog');
        if ($project_id) {
            $action->setValue('project_id', $project_id);
        }
        if (!empty($requestData['session_id'])) {
            $action->setValue('session_id', $requestData['session_id']);
        }
        $model = $responseData['model'] ?? ($requestData['model'] ?? null);
        if (!empty($model)) {
            $action->setValue('model', $model);
        }
        $action->save();
    }

    private function logErrorInteraction($project_id, $requestData, $error)
    {
        // Extract atomic error log data
        $atomicPayload = [
            'project_id' => $project_id,
            'session_id' => $requestData['session_id'] ?? null,
            'model' => $requestData['model'] ?? 'unknown',
            'timestamp' => date('Y-m-d H:i:s'),
            'error' => true,
            'error_type' => $error['type'] ?? 'UNKNOWN',
            'error_message' => $error['message'] ?? 'Unknown error'
        ];

        // Extract the last user message that caused the error
        if (!empty($requestData['messages']) && is_array($requestData['messages'])) {
            $userMessages = array_filter($requestData['messages'], function($msg) {
                return ($msg['role'] ?? '') === 'user';
            });
            $lastUserMessage = !empty($userMessages) ? array_values($userMessages)[count($userMessages) - 1] : null;

            if ($lastUserMessage) {
                $atomicPayload['user_message'] = $lastUserMessage['content'] ?? '';
            }
        }

        $action = new SecureChatLog($this);
        $action->setValue('message', json_encode($atomicPayload));
        $action->setValue('record', 'SecureChatLogError');
        if ($project_id) {
            $action->setValue('project_id', $project_id);
        }
        if (!empty($requestData['session_id'])) {
            $action->setValue('session_id', $requestData['session_id']);
        }
        $model = $requestData['model'] ?? null;
        if (!empty($model)) {
            $action->setValue('model', $model);
        }
        $action->save();
    }

    public function extractResponseText($response)
    {
        // Return structured output as-is if available (already JSON)
        if (isset($response['structured_output'])) {
            return $response['structured_output'];
        }
        return $response['content'] ?? json_encode($response);
    }


    public function extractUsageTokens($response)
    {
        return $response['usage'] ?? ['prompt_tokens' => 0, 'completion_tokens' => 0, 'total_tokens' => 0];
    }

    public function extractMetaData($response)
    {
        return [
            'id' => $response['id'] ?? 'N/A',
            'object' => $response['object'] ?? 'N/A',
            'created' => $response['created'] ?? 'N/A',
            'model' => $response['model'] ?? 'N/A',
            'usage' => $response['usage'] ?? 'N/A'
        ];
    }

    public function getGuzzleClient()
    {
        if (!$this->guzzleClient) {
            $options = [
                'timeout' => 30,
                'connect_timeout' => 10,
                'verify' => false
            ];

            // Apply DNS override if configured (for site-to-site VPN routing)
            $dnsOverrideIp = $this->getSystemSetting('apim_dns_override_ip');
            if (!empty($dnsOverrideIp)) {
                $options['curl'] = [
                    CURLOPT_RESOLVE => ["aihubapi.stanfordhealthcare.org:443:{$dnsOverrideIp}"],
                ];
            }

            $this->guzzleClient = new Client($options);
        }
        return $this->guzzleClient;
    }

    public function setGuzzleTimeout(float $timeout)
    {
        $this->guzzleTimeout = $timeout;
    }

    public function getGuzzleTimeout(): float
    {
        return $this->guzzleTimeout;
    }

    /**
     * This is the primary ajax handler for JSMO calls
     * @param $action
     * @param $payload
     * @param $project_id
     * @return array|array[]|bool
     * @throws Exception
     */

    public function redcap_module_api($action = null, $payload = [])
    {
        if (empty($action) && isset($_POST['action'])) {
            $action = $_POST['action'];
        }

        // Normalize payload from JSON or POST
        if (empty($payload)) {
            $raw = file_get_contents("php://input");
            $decoded = $raw ? json_decode($raw, true) : [];
            $payload = (json_last_error() === JSON_ERROR_NONE) ? $decoded : $_POST;
        }

        switch ($action) {
            case "callAI":
                $prompt = $payload['prompt'] ?? null;
                $model = $payload['model'] ?? 'deepseek';
                $temperature = isset($payload['temperature']) ? (float)$payload['temperature'] : 0.2;
                $max_tokens = isset($payload['max_tokens']) ? (int)$payload['max_tokens'] : 8000;
                $json_schema = $payload['json_schema'] ?? null;

                if (!$prompt) {
                    return [
                        "status"  => 400,
                        "body"    => json_encode(["error" => "Missing prompt"]),
                        "headers" => ["Content-Type" => "application/json"]
                    ];
                }

                $params = [
                    'messages' => [['role' => 'user', 'content' => $prompt]],
                    'temperature' => $temperature,
                    'max_tokens' => $max_tokens
                ];

                if ($json_schema !== null) {
                    $decoded = json_decode($json_schema, true);
                    if (json_last_error() === JSON_ERROR_NONE) {
                        $params['json_schema'] = $decoded;
                    }
                }

                $result = $this->callAI($model, $params);

                $response = [
                    'status' => 'success',
                    'model' => $model,
                    'content' => $this->extractResponseText($result),
                    'usage' => $this->extractUsageTokens($result)
                ];

                return [
                    "status"  => 200,
                    "body"    => json_encode($response),
                    "headers" => ["Content-Type" => "application/json"]
                ];

            case "messages":
                // Claude Messages API compatibility endpoint
                // Accepts: {model, messages, max_tokens, temperature, system, top_p, stop}
                // Returns: Claude Messages API format response

                $model = $payload['model'] ?? 'o3-mini';
                $messages = $payload['messages'] ?? null;

                // Handle messages as JSON string (from form-encoded POST)
                if (is_string($messages)) {
                    $decoded = json_decode($messages, true);
                    if (json_last_error() === JSON_ERROR_NONE && is_array($decoded)) {
                        $messages = $decoded;
                    }
                }

                $max_tokens = isset($payload['max_tokens']) ? (int)$payload['max_tokens'] : 4096;
                $temperature = isset($payload['temperature']) ? (float)$payload['temperature'] : null;
                $top_p = isset($payload['top_p']) ? (float)$payload['top_p'] : null;
                $stop = $payload['stop'] ?? null;
                $system = $payload['system'] ?? null;

                if (!$messages || !is_array($messages)) {
                    return [
                        "status"  => 400,
                        "body"    => json_encode([
                            "type" => "error",
                            "error" => [
                                "type" => "invalid_request_error",
                                "message" => "Missing or invalid 'messages' field"
                            ]
                        ]),
                        "headers" => ["Content-Type" => "application/json"]
                    ];
                }

                // Map Claude model names to SecureChatAI aliases
                $modelMap = [
                    'claude-3-7-sonnet-20250219' => 'claude',
                    'claude-3-5-sonnet-20241022' => 'claude',
                    'claude-sonnet-4' => 'claude-sonnet-4',
                    'gpt-4-1' => 'gpt-4-1',
                    'o1' => 'o1',
                    'o3-mini' => 'o3-mini',
                    'gpt-5' => 'gpt-5',
                    'llama3370b' => 'llama3370b',
                    'deepseek' => 'deepseek'
                ];
                $mappedModel = $modelMap[$model] ?? $model;

                // Prepend system message if provided
                if ($system) {
                    array_unshift($messages, [
                        'role' => 'system',
                        'content' => $system
                    ]);
                }

                // Build params for callAI()
                $params = ['messages' => $messages];

                if ($temperature !== null) $params['temperature'] = $temperature;
                if ($top_p !== null) $params['top_p'] = $top_p;
                if ($stop !== null) $params['stop'] = $stop;

                // Handle max_tokens vs max_completion_tokens
                if (in_array($mappedModel, ['o1', 'o3-mini', 'gpt-5'])) {
                    $params['max_completion_tokens'] = $max_tokens;
                } else {
                    $params['max_tokens'] = $max_tokens;
                }

                // Call the AI (no project_id for external API calls)
                $result = $this->callAI($mappedModel, $params, null);

                // Check for errors
                if (isset($result['error']) && $result['error']) {
                    return [
                        "status"  => 500,
                        "body"    => json_encode([
                            "type" => "error",
                            "error" => [
                                "type" => $result['type'] ?? "api_error",
                                "message" => $result['message'] ?? "Internal error"
                            ]
                        ]),
                        "headers" => ["Content-Type" => "application/json"]
                    ];
                }

                // Transform to Claude Messages API response format
                $content = $this->extractResponseText($result);
                $usage = $this->extractUsageTokens($result);

                // Determine stop_reason
                $stop_reason = "end_turn";
                if (isset($result['usage']['completion_tokens']) &&
                    isset($params['max_tokens']) &&
                    $result['usage']['completion_tokens'] >= $params['max_tokens']) {
                    $stop_reason = "max_tokens";
                }

                $claudeResponse = [
                    "id" => "msg_" . uniqid(),
                    "type" => "message",
                    "role" => "assistant",
                    "content" => [
                        [
                            "type" => "text",
                            "text" => is_string($content) ? $content : json_encode($content)
                        ]
                    ],
                    "model" => $model,  // Echo back the requested model name
                    "stop_reason" => $stop_reason,
                    "stop_sequence" => null,
                    "usage" => [
                        "input_tokens" => $usage['prompt_tokens'] ?? 0,
                        "output_tokens" => $usage['completion_tokens'] ?? 0
                    ]
                ];

                return [
                    "status"  => 200,
                    "body"    => json_encode($claudeResponse),
                    "headers" => ["Content-Type" => "application/json"]
                ];

            case "getSession":
                $session_id = $payload['session_id'] ?? null;
                $project_id = $payload['project_id'] ?? null;

                if (!$session_id) {
                    return [
                        "status"  => 400,
                        "body"    => json_encode(["error" => "Missing session_id"]),
                        "headers" => ["Content-Type" => "application/json"]
                    ];
                }

                // Use the same rehydration path as the chatbot (prefers clean
                // CappyChatTurn rows, backfills pre-turn legacy history) so the
                // two "rehydrate a session" entry points don't diverge. This is
                // the token-API path (no browser user), so it is not user-scoped
                // — the API token is the authorization boundary here.
                $session = $this->rehydrateProjectSession($session_id, $project_id);

                return [
                    "status"  => 200,
                    "body"    => json_encode($session),
                    "headers" => ["Content-Type" => "application/json"]
                ];

            default:
                return [
                    "status"  => 400,
                    "body"    => json_encode(["error" => "Action $action not defined"]),
                    "headers" => ["Content-Type" => "application/json"]
                ];
        }
    }
}
?>