mkdocs.yml

site_name: Ragas
site_description: Evaluation framework for your AI Application
site_url: !ENV READTHEDOCS_CANONICAL_URL
repo_name: vibrantlabsai/ragas
repo_url: https://github.com/vibrantlabsai/ragas
watch:
  - src

# Navigation
nav:
  - "": index.md
  - 🚀 Get Started:
      - getstarted/index.md
      - Installation: getstarted/install.md
      - Quick Start: getstarted/quickstart.md
      - Tutorials:
          - Evaluate a prompt: tutorials/prompt.md
          - Evaluate a simple RAG system: tutorials/rag.md
          - Evaluate an AI Workflow: tutorials/workflow.md
          - Evaluate an AI Agent: tutorials/agent.md
  - 📚 Core Concepts:
      - concepts/index.md
      - Experimentation: concepts/experimentation.md
      - Datasets: concepts/datasets.md
      - Metrics:
          - concepts/metrics/index.md
          - Overview: concepts/metrics/overview/index.md
          - Available Metrics:
              - concepts/metrics/available_metrics/index.md
              - Retrieval Augmented Generation:
                  - Context Precision: concepts/metrics/available_metrics/context_precision.md
                  - Context Recall: concepts/metrics/available_metrics/context_recall.md
                  - Context Entities Recall: concepts/metrics/available_metrics/context_entities_recall.md
                  - Noise Sensitivity: concepts/metrics/available_metrics/noise_sensitivity.md
                  - Response Relevancy: concepts/metrics/available_metrics/answer_relevance.md
                  - Faithfulness: concepts/metrics/available_metrics/faithfulness.md
              - Nvidia Metrics:
                  - Answer Accuracy: concepts/metrics/available_metrics/nvidia_metrics/#answer-accuracy
                  - Context Relevance: concepts/metrics/available_metrics/nvidia_metrics/#context-relevance
                  - Response Groundedness: concepts/metrics/available_metrics/nvidia_metrics/#response-groundedness
              - Agents or Tool Use Cases:
                    - concepts/metrics/available_metrics/agents.md
                    - Topic Adherence: concepts/metrics/available_metrics/agents/#topic-adherence
                    - Tool Call Accuracy: concepts/metrics/available_metrics/agents/#tool-call-accuracy
                    - Tool Call F1: concepts/metrics/available_metrics/agents/#tool-call-f1
                    - Agent Goal Accuracy: concepts/metrics/available_metrics/agents/#agent-goal-accuracy
              - Natural Language Comparison:
                  - Factual Correctness: concepts/metrics/available_metrics/factual_correctness.md
                  - Semantic Similarity: concepts/metrics/available_metrics/semantic_similarity.md
                  - Traditional non LLM metrics:
                    - concepts/metrics/available_metrics/traditional.md
                    - Non LLM String Similarity: concepts/metrics/available_metrics/traditional/#non-llm-string-similarity
                    - BLEU Score: concepts/metrics/available_metrics/traditional/#bleu-score
                    - CHRF Score: concepts/metrics/available_metrics/traditional/#chrf-score
                    - ROUGE Score: concepts/metrics/available_metrics/traditional/#rouge-score
                    - String Presence: concepts/metrics/available_metrics/traditional/#string-presence
                    - Exact Match: concepts/metrics/available_metrics/traditional/#exact-match
              - SQL:
                  - concepts/metrics/available_metrics/sql.md
                  - Execution based Datacompy Score: concepts/metrics/available_metrics/sql/#execution-based-metrics
                  - SQL Query Equivalence: concepts/metrics/available_metrics/sql/#sql-query-semantic-equivalence
              - General Purpose:
                  - concepts/metrics/available_metrics/general_purpose.md
                  - Aspect Critic: concepts/metrics/available_metrics/general_purpose/#aspect-critic
                  - Simple Criteria Scoring: concepts/metrics/available_metrics/general_purpose/#simple-criteria-scoring
                  - Rubrics Based Scoring: concepts/metrics/available_metrics/general_purpose/#rubrics-based-criteria-scoring
                  - Instance Specific Rubrics Scoring: concepts/metrics/available_metrics/general_purpose/#instance-specific-rubrics-criteria-scoring
              - Other Tasks:
                  - Summarization: concepts/metrics/available_metrics/summarization_score.md
      - Test Data Generation:
          - concepts/test_data_generation/index.md
          - RAG:
              - concepts/test_data_generation/rag.md
              - KG Building: concepts/test_data_generation/rag/#knowledge-graph-creation
              - Scenario Generation: concepts/test_data_generation/rag/#scenario-generation
          - Agents or tool use:
              - concepts/test_data_generation/agents.md
      - Components:
          - concepts/components/index.md
          - General:
              - Prompt: concepts/components/prompt.md
          - Evaluation:
              - Evaluation Sample: concepts/components/eval_sample.md
              - Evaluation Dataset: concepts/components/eval_dataset.md

  - 🛠️ How-to Guides:
      - howtos/index.md
      - Customizations:
          - howtos/customizations/index.md
          - General:
              - Customise models: howtos/customizations/customize_models.md
              - Run Config: howtos/customizations/run_config.md
              - Caching: howtos/customizations/_caching.md
              - Cancelling Tasks: howtos/customizations/cancellation.md
          - LLM Adapters: howtos/llm-adapters.md
          - Metrics:
              - Modify Prompts: howtos/customizations/metrics/modifying-prompts-metrics.md
              - Adapt Metrics to Languages: howtos/customizations/metrics/_metrics_language_adaptation.md
              - Train and Align Metrics: howtos/customizations/metrics/train_your_own_metric.md
          - Testset Generation:
              - Non-English Testset Generation: howtos/customizations/testgenerator/_language_adaptation.md
              - Persona Generation: howtos/customizations/testgenerator/_persona_generator.md
              - Custom Single-hop Query: howtos/customizations/testgenerator/_testgen-custom-single-hop.md
              - Custom Multi-hop Query: howtos/customizations/testgenerator/_testgen-customisation.md
              - Using Pre-chunked Data: howtos/customizations/testgenerator/prechunked_data.md
          - Optimizers:
              - DSPy Optimizer: howtos/customizations/optimizers/index.md

      - Applications:
          - howtos/applications/index.md
          - Prompt Evaluation:
            - Iterate and Improve Prompts: howtos/applications/iterate_prompt.md
            - Systematic Prompt Optimization: howtos/applications/prompt_optimization.md
          - Metrics:
            - Cost Analysis: howtos/applications/_cost.md
            - Evaluating Multi-turn Conversations: howtos/applications/evaluating_multi_turn_conversations.md
            - Evaluations with Vertex AI models: howtos/applications/vertexai_x_ragas.md
          - Testset Generation:
            - Single-hop Query Testset: howtos/applications/singlehop_testset_gen.md
          - Benchmarking:
            - Evaluate a New LLM: howtos/applications/benchmark_llm.md
          - Agent Evaluation:
            - Evaluate a Text-to-SQL Agent: howtos/applications/text2sql.md
            - Align an LLM as a Judge: howtos/applications/align-llm-as-judge.md
          - RAG Evaluation:
            - Evaluate and Improve a RAG App: howtos/applications/evaluate-and-improve-rag.md
      - CLI:
          - howtos/cli/index.md
          - RAG Evaluation: howtos/cli/rag_eval.md
          - Improve RAG: howtos/cli/improve_rag.md
      - Integrations:
          - howtos/integrations/index.md
          - Observability:
              - Arize: howtos/integrations/_arize.md
              - LangSmith: howtos/integrations/langsmith.md
          - LLM Providers:
              - Amazon Bedrock: howtos/integrations/amazon_bedrock.md
              - Google Gemini: howtos/integrations/gemini.md
              - OCI Gen AI: howtos/integrations/oci_genai.md
          - Frameworks:
              - AG-UI: howtos/integrations/ag_ui.md
              - Griptape: howtos/integrations/griptape.md
              - Haystack: howtos/integrations/haystack.md
              - LangChain: howtos/integrations/langchain.md
              - LangGraph: howtos/integrations/_langgraph_agent_evaluation.md
              - LlamaIndex: howtos/integrations/_llamaindex.md
              - LlamaIndex Agents: howtos/integrations/llamaindex_agents.md
              - LlamaStack: howtos/integrations/llama_stack.md
              - R2R: howtos/integrations/r2r.md
              - Swarm: howtos/integrations/swarm_agent_evaluation.md
      - Migrations:
          - From v0.1 to v0.2: howtos/migrations/migrate_from_v01_to_v02.md
          - From v0.3 to v0.4: howtos/migrations/migrate_from_v03_to_v04.md
  - 📖 References:
    - references/index.md
    - Core:
      - Prompt: references/prompt.md
      - LLMs: references/llms.md
      - Embeddings: references/embeddings.md
      - Tokenizers: references/tokenizers.md
      - RunConfig: references/run_config.md
      - Executor: references/executor.md
      - Cache: references/cache.md
      - Optimizers: references/optimizers.md
    - Evaluation:
      - Schemas: references/evaluation_schema.md
      - Metrics: references/metrics.md
      - evaluate(): references/evaluate.md
      - aevaluate(): references/aevaluate.md
    - Testset Generation:
      - Schemas: references/testset_schema.md
      - Graph: references/graph.md
      - Transforms: references/transforms.md
      - Synthesizers: references/synthesizers.md
      - Generation: references/generate.md
    - Integrations: references/integrations.md
  - ❤️ Community: community/index.md

# https://www.mkdocs.org/user-guide/configuration/#validation
validation:
  omitted_files: warn
  absolute_links: warn
  unrecognized_links: warn

# Material-Docs Theme
theme:
  name: material
  custom_dir: docs/extra/overrides
  logo: _static/imgs/ragas-logo.png
  favicon: _static/favicon.ico
  features:
    - announce.dismiss
    - content.tabs.link
    - content.code.annotate
    - content.code.copy
    - announce.dismiss
    - navigation.tabs
    - navigation.path
    - navigation.instant
    - navigation.instant.prefetch
    - navigation.instant.preview
    - navigation.sections
    - navigation.top
    - navigation.tracking
    - navigation.indexes
    - navigation.footer
    - search.suggest
    - search.highlight
  palette:
    - media: "(prefers-color-scheme)"
      toggle:
        icon: material/brightness-auto
        name: Switch to light mode
    - media: "(prefers-color-scheme: light)"
      scheme: default
      primary: "#bd8526"
      accent: "#bd8526"
      toggle:
        icon: material/brightness-7
        name: Switch to dark mode
    - media: "(prefers-color-scheme: dark)"
      scheme: slate
      primary: "#bd8526"
      accent: "#bd8526"
      toggle:
        icon: material/brightness-4
        name: Switch to system preference

markdown_extensions:
  - pymdownx.highlight:
      anchor_linenums: true
      line_spans: __span
      pygments_lang_class: true
  - admonition
  - pymdownx.inlinehilite
  - pymdownx.details
  - pymdownx.tabbed:
      alternate_style: true
  - pymdownx.emoji:
      emoji_index: !!python/name:material.extensions.emoji.twemoji
      emoji_generator: !!python/name:material.extensions.emoji.to_svg
  - attr_list
  - md_in_html
  - pymdownx.arithmatex:
      generic: true
  - pymdownx.superfences:
      custom_fences:
        - name: mermaid
          class: mermaid
          format: !!python/name:pymdownx.superfences.fence_code_format
  - pymdownx.snippets:
      base_path: ["./docs/extra/components/"]

# Extra CSS
extra_css:
  - extra/ragas-modern.css

# Plugins
extra:
  version:
    provider: mike
  analytics:
    provider: google
    property: !ENV GOOGLE_ANALYTICS_KEY
plugins:
  - search
  - social:
      enabled: !ENV [MKDOCS_CI, true]
  - copy-to-llm:
      repo_url: "https://raw.githubusercontent.com/vibrantlabsai/ragas/main/docs"
      buttons:
        copy_page: true
        copy_markdown_link: false  # Disabled until plugin bug is fixed
        view_as_markdown: false    # Disabled until plugin bug is fixed
        open_in_chatgpt: true
        open_in_claude: true
  - llmstxt:
      markdown_description: |
        Ragas is an open-source evaluation framework for LLM applications including RAG pipelines,
        AI agents, and workflows. It provides objective metrics for evaluation, test data generation
        capabilities, and integrations with popular LLM frameworks like LangChain and LlamaIndex.
      full_output: llms-full.txt
      sections:
        Getting Started:
          - getstarted/*.md
        Tutorials:
          - tutorials/*.md
        Core Concepts:
          - concepts/*.md
          - concepts/components/*.md
        Metrics:
          - concepts/metrics/overview/*.md
          - concepts/metrics/available_metrics/*.md
        Test Data Generation:
          - concepts/test_data_generation/*.md
        Customization Guides:
          - howtos/customizations/*.md
          - howtos/customizations/metrics/*.md
          - howtos/customizations/testgenerator/*.md
          - howtos/customizations/optimizers/*.md
        Application Guides:
          - howtos/applications/*.md
        CLI:
          - howtos/cli/*.md
        Integrations:
          - howtos/integrations/*.md
        API Reference:
          - references/*.md
  - git-revision-date-localized:
      enabled: !ENV [MKDOCS_CI, false]
      enable_creation_date: true
  - git-committers:
      enabled: !ENV [MKDOCS_CI, false]
      repository: vibrantlabsai/ragas
      branch: main
  - mkdocstrings:
      handlers:
        python:
          paths: [src]
          options:
            docstring_style: numpy
            members_order: source
            separate_signature: true
            filters: ["!^_"]
            docstring_options:
              ignore_init_summary: true
            merge_init_into_class: true
            show_signature_annotations: true
            signature_crossrefs: true
  - glightbox
  # - gen-files:
  #     scripts:
  #       - docs/ipynb_to_md.py
extra_javascript:
  - _static/js/mathjax.js
  - _static/js/header_border.js
  - https://unpkg.com/mathjax@3/es5/tex-mml-chtml.js
  - _static/js/toggle.js
  - https://cdn.octolane.com/tag.js?pk=c7c9b2b863bf7eaf4e2a # octolane for analytics
  - _static/js/commonroom.js # commonroom analytics