{
  "name": "Evaluate AI agent response relevance using OpenAI and cosine similarity",
  "nodes": [
    {
      "id": "49b274c7-1ff2-43ba-acb9-349a8665b513",
      "name": "OpenAI Chat Model1",
      "type": "@n8n/n8n-nodes-langchain.lmChatOpenAi",
      "position": [
        148,
        180
      ]
    },
    {
      "id": "77f0c130-085f-4fa5-b74e-66fd084949b7",
      "name": "When fetching a dataset row",
      "type": "n8n-nodes-base.evaluationTrigger",
      "position": [
        -380,
        -140
      ]
    },
    {
      "id": "63d8ff44-5c4b-4cf6-b5b2-1f22de0de495",
      "name": "Remap Input",
      "type": "n8n-nodes-base.set",
      "position": [
        -160,
        -140
      ]
    },
    {
      "id": "d8ac1b2b-f4ee-4af1-92f5-d194832c5c76",
      "name": "Evaluation",
      "type": "n8n-nodes-base.evaluation",
      "position": [
        436,
        -40
      ]
    },
    {
      "id": "1d8d9c97-0c90-4878-95fa-0dc6d3ad82c6",
      "name": "Set Input Fields",
      "type": "n8n-nodes-base.set",
      "position": [
        656,
        -140
      ]
    },
    {
      "id": "d4ad7c5e-f1cf-463b-b9b3-a45ff5597199",
      "name": "No Operation, do nothing",
      "type": "n8n-nodes-base.noOp",
      "position": [
        656,
        60
      ]
    },
    {
      "id": "1aea189f-3e7d-45d2-9802-d44b1dce4797",
      "name": "AI Agent",
      "type": "@n8n/n8n-nodes-langchain.agent",
      "position": [
        60,
        -40
      ]
    },
    {
      "id": "54596db1-4ba7-414c-88cf-250f528d6c21",
      "name": "When chat message received",
      "type": "@n8n/n8n-nodes-langchain.chatTrigger",
      "position": [
        -160,
        60
      ]
    },
    {
      "id": "4184f202-48f5-4e58-a0e8-0227a23553f3",
      "name": "Update Output",
      "type": "n8n-nodes-base.evaluation",
      "position": [
        2132,
        -140
      ]
    },
    {
      "id": "fe7b41d2-e382-4f59-8b0c-06361062581d",
      "name": "Update Metrics",
      "type": "n8n-nodes-base.evaluation",
      "position": [
        2352,
        -140
      ]
    },
    {
      "id": "04f714cd-99ce-45a9-bc43-351fb6ea14d4",
      "name": "OpenAI Chat Model",
      "type": "@n8n/n8n-nodes-langchain.lmChatOpenAi",
      "position": [
        920,
        80
      ]
    },
    {
      "id": "d821b0ec-93c9-46e1-ab03-280aeb9b87b1",
      "name": "Structured Output Parser",
      "type": "@n8n/n8n-nodes-langchain.outputParserStructured",
      "position": [
        1040,
        80
      ]
    },
    {
      "id": "be975ef4-c50b-4bf9-b871-cc8d479661eb",
      "name": "Answer Relevance",
      "type": "@n8n/n8n-nodes-langchain.chainLlm",
      "position": [
        900,
        -140
      ]
    },
    {
      "id": "8e3df14a-7754-4257-8209-a24cfa30dbbc",
      "name": "Sticky Note1",
      "type": "n8n-nodes-base.stickyNote",
      "position": [
        -452,
        -400
      ],
      "parameters": {
        "width": 840,
        "height": 720,
        "content": "## 1. Setup Your AI Workflow to Use Evaluations\n[Learn more about the Evaluations Trigger](https://docs.n8n.io/integrations/builtin/?utm_source=n8n_app&utm_medium=node_settings_modal-credential_link&u"
      }
    },
    {
      "id": "eeeac9d0-39c6-4d17-8d93-9c60b6c612f8",
      "name": "Sticky Note",
      "type": "n8n-nodes-base.stickyNote",
      "position": [
        820,
        -400
      ],
      "parameters": {
        "width": 1200,
        "height": 720,
        "content": "## 2. Answer Relevance: How relevant is the agent response to the user question?\n[Learn more about the Evaluations Trigger](https://docs.n8n.io/integrations/builtin/?utm_source=n8n_app&utm_medium=node"
      }
    },
    {
      "id": "c86f66e2-9997-4ca6-9ec6-cfa185d404f2",
      "name": "Sticky Note3",
      "type": "n8n-nodes-base.stickyNote",
      "position": [
        -920,
        -340
      ],
      "parameters": {
        "width": 420,
        "height": 720,
        "content": "## Try It Out!\n### This n8n template demonstrates how to calculate the evaluation metric \"Relevance\" which in this scenario, measures the relevance of the agent's response to the user's question.\n\nThe"
      }
    },
    {
      "id": "3bdd3254-7cdf-455a-ae63-9639069993bc",
      "name": "Get Embeddings",
      "type": "n8n-nodes-base.httpRequest",
      "position": [
        1460,
        -140
      ]
    },
    {
      "id": "9e3bcdee-5f24-45e6-a85f-6dcfacb58d5d",
      "name": "Calculate Similarity Score",
      "type": "n8n-nodes-base.code",
      "position": [
        1660,
        -140
      ]
    },
    {
      "id": "69c30814-9725-426d-9572-9f92bd81dea5",
      "name": "Questions to Items",
      "type": "n8n-nodes-base.code",
      "position": [
        1260,
        -140
      ]
    },
    {
      "id": "83be23ff-6857-4265-b1d0-f4b682c66903",
      "name": "Calculate Relevance Score",
      "type": "n8n-nodes-base.set",
      "position": [
        1860,
        -140
      ]
    }
  ],
  "connections": {
    "AI Agent": {
      "main": [
        [
          {
            "node": "Evaluation",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Evaluation": {
      "main": [
        [
          {
            "node": "Set Input Fields",
            "type": "main",
            "index": 0
          }
        ],
        [
          {
            "node": "No Operation, do nothing",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Remap Input": {
      "main": [
        [
          {
            "node": "AI Agent",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Update Output": {
      "main": [
        [
          {
            "node": "Update Metrics",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Get Embeddings": {
      "main": [
        [
          {
            "node": "Calculate Similarity Score",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Answer Relevance": {
      "main": [
        [
          {
            "node": "Questions to Items",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Set Input Fields": {
      "main": [
        [
          {
            "node": "Answer Relevance",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "OpenAI Chat Model": {
      "ai_languageModel": [
        [
          {
            "node": "Answer Relevance",
            "type": "ai_languageModel",
            "index": 0
          }
        ]
      ]
    },
    "OpenAI Chat Model1": {
      "ai_languageModel": [
        [
          {
            "node": "AI Agent",
            "type": "ai_languageModel",
            "index": 0
          }
        ]
      ]
    },
    "Questions to Items": {
      "main": [
        [
          {
            "node": "Get Embeddings",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Structured Output Parser": {
      "ai_outputParser": [
        [
          {
            "node": "Answer Relevance",
            "type": "ai_outputParser",
            "index": 0
          }
        ]
      ]
    },
    "Calculate Relevance Score": {
      "main": [
        [
          {
            "node": "Update Output",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Calculate Similarity Score": {
      "main": [
        [
          {
            "node": "Calculate Relevance Score",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "When chat message received": {
      "main": [
        [
          {
            "node": "AI Agent",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "When fetching a dataset row": {
      "main": [
        [
          {
            "node": "Remap Input",
            "type": "main",
            "index": 0
          }
        ]
      ]
    }
  }
}