Support vllm style --tool-call-parser for hermes, qwen35, gemma4

.gitignore

@@ -30,5 +30,6 @@ site/
 build/
 dist/
 /models/**/*
+/.scratchpad/
 .scratchpad/
 .plans/

config.yaml

@@ -14,6 +14,7 @@ models:
     model_type: llm
     model_path: /mnt/Ironwolf-4TB/Models/OpenVINO/Qwen3.5/Qwen3.5-0.8B-int8_asym-ov/
     device: CPU
+    tool_call_parser: qwen35
     runtime_config:
       PERFORMANCE_HINT: LATENCY
     scheduler_config:

src/cli/groups/add.py

@@ -62,8 +62,13 @@
     default=None,
     type=float,
     help='Confidence threshold for accepting draft tokens.')
+@click.option('--tool-call-parser',
+    type=click.Choice(['qwen35', 'hermes', 'gemma4']),
+    required=False,
+    default=None,
+    help='Tool-call output format for this model (qwen35 XML, hermes JSON, or gemma4 call syntax). llm/vlm only; required for tool calling.')
 @click.pass_context
-def add(ctx, model_path, model_name, engine, model_type, device, runtime_config, scheduler_config, cache_dir, draft_model_path, draft_device, num_assistant_tokens, assistant_confidence_threshold):
+def add(ctx, model_path, model_name, engine, model_type, device, runtime_config, scheduler_config, cache_dir, draft_model_path, draft_device, num_assistant_tokens, assistant_confidence_threshold, tool_call_parser):
     """- Add a model configuration to the config file."""
 
     # Validate model path
@@ -128,6 +133,8 @@ def add(ctx, model_path, model_name, engine, model_type, device, runtime_config,
         load_config["num_assistant_tokens"] = num_assistant_tokens
     if assistant_confidence_threshold is not None:
         load_config["assistant_confidence_threshold"] = assistant_confidence_threshold
+    if tool_call_parser:
+        load_config["tool_call_parser"] = tool_call_parser
 
     ctx.obj.server_config.save_model_config(model_name, load_config)
     console.print(f"[green]Model configuration saved:[/green] {model_name}")

src/engine/ov_genai/llm.py

@@ -16,7 +16,7 @@
 from src.server.schemas.modeling.contract_ovgenai_llm_and_vlm import OVGenAI_GenConfig
 from src.server.model_registry import ModelRegistry
 from src.server.schemas.registration import ModelLoadConfig
-from src.engine.ov_genai.streamers import ChunkStreamer
+from src.engine.ov_genai.streamers import ensure_tool_call_parser, select_streamer
 from src.server.utils.chat import flatten_messages
 
 logger = logging.getLogger(__name__)
@@ -83,6 +83,7 @@ async def generate_text(self, gen_config: OVGenAI_GenConfig) -> AsyncIterator[Un
         Async non-streaming text generation.
         Yields in order: metrics (dict), new_text (str).
         """
+        ensure_tool_call_parser(gen_config, self.load_config)
         generation_kwargs = self.create_generation_config(gen_config)
 
         # Support pre-encoded input_ids, raw prompts, and chat messages
@@ -101,7 +102,13 @@ async def generate_text(self, gen_config: OVGenAI_GenConfig) -> AsyncIterator[Un
 
         perf_metrics = result.perf_metrics
         decoder_tokenizer = self.model.get_tokenizer()
-        text = decoder_tokenizer.decode(result.tokens)[0] if getattr(result, "tokens", None) else ""
+        # gemma4 protocol tags are special=True: keep them in the decoded text
+        # so the route-level parse_generation can split reasoning/tool calls.
+        keep_special = getattr(gen_config, "tool_call_parser", None) == "gemma4"
+        text = (
+            decoder_tokenizer.decode(result.tokens, skip_special_tokens=not keep_special)[0]
+            if getattr(result, "tokens", None) else ""
+        )
 
         metrics_dict = self.collect_metrics(gen_config, perf_metrics)
         yield metrics_dict
@@ -112,10 +119,10 @@ async def generate_stream(self, gen_config: OVGenAI_GenConfig) -> AsyncIterator[
         Async streaming text generation.
         Yields token chunks (str) as they arrive, then metrics (dict), then final new_text (str).
         """
-
+        ensure_tool_call_parser(gen_config, self.load_config)
         generation_kwargs = self.create_generation_config(gen_config)
         decoder_tokenizer = self.model.get_tokenizer()
-        streamer = ChunkStreamer(decoder_tokenizer, gen_config)
+        streamer = select_streamer(decoder_tokenizer, gen_config)
 
         # Track active request and streamer for cancellation
         self._active_request_id = gen_config.request_id