{
  "name": "Evaluation metric example: RAG document relevance",
  "nodes": [
    {
      "id": "86d6a658-f056-4434-9720-dfd3ca514cb3",
      "name": "Sticky Note1",
      "type": "n8n-nodes-base.stickyNote",
      "position": [
        420,
        20
      ],
      "parameters": {
        "width": 300,
        "height": 180,
        "content": "Check whether the documents returned a relevant to the question"
      }
    },
    {
      "id": "90aac144-164f-4961-89f8-4b8402177202",
      "name": "Sticky Note3",
      "type": "n8n-nodes-base.stickyNote",
      "position": [
        -1220,
        -40
      ],
      "parameters": {
        "width": 200,
        "height": 500,
        "content": "## How it works\nThis template shows how to calculate a workflow evaluation metric: **retrieved document relevance** (i.e. whether the information retrieved from the vector store is relevant to the que"
      }
    },
    {
      "id": "f3970d8c-b205-4348-b375-d919d2f56f94",
      "name": "Sticky Note4",
      "type": "n8n-nodes-base.stickyNote",
      "position": [
        -960,
        280
      ],
      "parameters": {
        "width": 220,
        "height": 220,
        "content": "Read in [this test dataset](https://docs.google.com/spreadsheets/d/1uuPS5cHtSNZ6HNLOi75A2m8nVWZrdBZ_Ivf58osDAS8/edit?gid=630157309#gid=630157309) of questions"
      }
    },
    {
      "id": "677b90c8-aa6f-4cc5-88df-d8e635b0bde2",
      "name": "When fetching a dataset row",
      "type": "n8n-nodes-base.evaluationTrigger",
      "position": [
        -900,
        340
      ]
    },
    {
      "id": "c381a3c3-12b6-44fe-a2ed-6adf6f997534",
      "name": "Evaluating?",
      "type": "n8n-nodes-base.evaluation",
      "position": [
        -60,
        200
      ]
    },
    {
      "id": "e1cf0122-c668-41d2-9316-a173d425d897",
      "name": "When chat message received",
      "type": "@n8n/n8n-nodes-langchain.chatTrigger",
      "position": [
        -900,
        100
      ]
    },
    {
      "id": "b521aa40-c8da-4485-b834-50a129c27d3c",
      "name": "Match chat format",
      "type": "n8n-nodes-base.set",
      "position": [
        -680,
        340
      ]
    },
    {
      "id": "d0170cbd-ba01-45db-a405-aec943bf1608",
      "name": "Sticky Note",
      "type": "n8n-nodes-base.stickyNote",
      "position": [
        -80,
        100
      ],
      "parameters": {
        "width": 150,
        "height": 260,
        "content": "Only calculate metrics if we're evaluating, to reduce costs"
      }
    },
    {
      "id": "a21f98f0-25eb-44cc-a04a-33181d7e5288",
      "name": "Return chat response",
      "type": "n8n-nodes-base.noOp",
      "position": [
        220,
        340
      ]
    },
    {
      "id": "fd8628a1-9f31-4f42-88df-b87eda7216ee",
      "name": "Set metrics",
      "type": "n8n-nodes-base.evaluation",
      "position": [
        820,
        80
      ]
    },
    {
      "id": "eefe3a5a-722a-4cce-b84b-65c0520a225f",
      "name": "Get dataset",
      "type": "n8n-nodes-base.googleSheets",
      "position": [
        -680,
        -620
      ]
    },
    {
      "id": "ea83d68c-f499-4864-918c-5ccaccf555a5",
      "name": "Remove Duplicates",
      "type": "n8n-nodes-base.removeDuplicates",
      "position": [
        -460,
        -620
      ]
    },
    {
      "id": "6bf59dc8-fe59-49cf-971f-e22dac5d0500",
      "name": "Simple Vector Store",
      "type": "@n8n/n8n-nodes-langchain.vectorStoreInMemory",
      "position": [
        -240,
        -620
      ]
    },
    {
      "id": "66dd59b0-b2e4-43c9-b1f5-7f5a65568f5e",
      "name": "Embeddings OpenAI",
      "type": "@n8n/n8n-nodes-langchain.embeddingsOpenAi",
      "position": [
        -300,
        -400
      ]
    },
    {
      "id": "213b230d-0f40-4e71-b0bf-4783274e2cdb",
      "name": "Default Data Loader",
      "type": "@n8n/n8n-nodes-langchain.documentDefaultDataLoader",
      "position": [
        -100,
        -380
      ]
    },
    {
      "id": "64ec7e8c-63f9-4198-9e8d-61345c564c13",
      "name": "Recursive Character Text Splitter",
      "type": "@n8n/n8n-nodes-langchain.textSplitterRecursiveCharacterTextSplitter",
      "position": [
        -20,
        -220
      ]
    },
    {
      "id": "c561f2ce-ef05-4f42-9b6e-6fabd12d6d23",
      "name": "Sticky Note2",
      "type": "n8n-nodes-base.stickyNote",
      "position": [
        -980,
        -700
      ],
      "parameters": {
        "width": 1180,
        "height": 620,
        "content": "### Setup: Populate vector DB\nRun this once before running the main workflow. It inserts the documents from the dataset in the vector store, so they can be queried by the agent below"
      }
    },
    {
      "id": "4a7de6bd-17ac-40df-9b27-2d8f6fdbeb07",
      "name": "When clicking ‘Execute workflow’",
      "type": "n8n-nodes-base.manualTrigger",
      "position": [
        -900,
        -620
      ]
    },
    {
      "id": "a0f6211f-6202-462c-8fa3-52e10ad89b5d",
      "name": "Calculate doc relevance metric",
      "type": "@n8n/n8n-nodes-langchain.openAi",
      "position": [
        440,
        80
      ]
    },
    {
      "id": "b01d3427-4907-4715-8c15-669500012f89",
      "name": "AI Agent",
      "type": "@n8n/n8n-nodes-langchain.agent",
      "position": [
        -420,
        200
      ]
    },
    {
      "id": "97e2db38-0114-449f-bb41-0ab490fc7327",
      "name": "OpenAI Chat Model",
      "type": "@n8n/n8n-nodes-langchain.lmChatOpenAi",
      "position": [
        -480,
        440
      ]
    },
    {
      "id": "5b3679ab-b0a6-4461-9276-89eddb16f699",
      "name": "Extract documents",
      "type": "n8n-nodes-base.set",
      "position": [
        220,
        80
      ]
    },
    {
      "id": "0e4c4512-7b9f-4881-8ac7-4c7c24a4a446",
      "name": "Simple Vector Store1",
      "type": "@n8n/n8n-nodes-langchain.vectorStoreInMemory",
      "position": [
        -260,
        440
      ]
    },
    {
      "id": "edde6c75-dea5-46cc-937e-68e4a925559b",
      "name": "Embeddings OpenAI1",
      "type": "@n8n/n8n-nodes-langchain.embeddingsOpenAi",
      "position": [
        -220,
        620
      ]
    },
    {
      "id": "e23289e7-a769-4aae-82e6-b7f9e6d903f0",
      "name": "Sticky Note6",
      "type": "n8n-nodes-base.stickyNote",
      "position": [
        -980,
        -40
      ],
      "parameters": {
        "width": 2000,
        "height": 820,
        "content": "### Main workflow"
      }
    },
    {
      "id": "c90baefc-22bd-4782-a8eb-236bfb1d2751",
      "name": "Sticky Note5",
      "type": "n8n-nodes-base.stickyNote",
      "position": [
        -440,
        120
      ],
      "parameters": {
        "width": 300,
        "height": 200,
        "content": "Make sure to enable 'Return intermediate steps' in the agent, to get the list of executed tools"
      }
    }
  ],
  "connections": {
    "AI Agent": {
      "main": [
        [
          {
            "node": "Evaluating?",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Evaluating?": {
      "main": [
        [
          {
            "node": "Extract documents",
            "type": "main",
            "index": 0
          }
        ],
        [
          {
            "node": "Return chat response",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Get dataset": {
      "main": [
        [
          {
            "node": "Remove Duplicates",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Embeddings OpenAI": {
      "ai_embedding": [
        [
          {
            "node": "Simple Vector Store",
            "type": "ai_embedding",
            "index": 0
          }
        ]
      ]
    },
    "Extract documents": {
      "main": [
        [
          {
            "node": "Calculate doc relevance metric",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Match chat format": {
      "main": [
        [
          {
            "node": "AI Agent",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "OpenAI Chat Model": {
      "ai_languageModel": [
        [
          {
            "node": "AI Agent",
            "type": "ai_languageModel",
            "index": 0
          }
        ]
      ]
    },
    "Remove Duplicates": {
      "main": [
        [
          {
            "node": "Simple Vector Store",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Embeddings OpenAI1": {
      "ai_embedding": [
        [
          {
            "node": "Simple Vector Store1",
            "type": "ai_embedding",
            "index": 0
          }
        ]
      ]
    },
    "Default Data Loader": {
      "ai_document": [
        [
          {
            "node": "Simple Vector Store",
            "type": "ai_document",
            "index": 0
          }
        ]
      ]
    },
    "Simple Vector Store1": {
      "ai_tool": [
        [
          {
            "node": "AI Agent",
            "type": "ai_tool",
            "index": 0
          }
        ]
      ]
    },
    "When chat message received": {
      "main": [
        [
          {
            "node": "AI Agent",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "When fetching a dataset row": {
      "main": [
        [
          {
            "node": "Match chat format",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Calculate doc relevance metric": {
      "main": [
        [
          {
            "node": "Set metrics",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Recursive Character Text Splitter": {
      "ai_textSplitter": [
        [
          {
            "node": "Default Data Loader",
            "type": "ai_textSplitter",
            "index": 0
          }
        ]
      ]
    },
    "When clicking ‘Execute workflow’": {
      "main": [
        [
          {
            "node": "Get dataset",
            "type": "main",
            "index": 0
          }
        ]
      ]
    }
  }
}