{
  "name": "Evaluation metric example: Correctness (judged by AI)",
  "nodes": [
    {
      "id": "4e2acf3b-3629-4719-b6dd-80e0efdd1cad",
      "name": "Sticky Note1",
      "type": "n8n-nodes-base.stickyNote",
      "position": [
        200,
        20
      ],
      "parameters": {
        "width": 300,
        "height": 180,
        "content": "Check whether the answer has the same meaning as the expected answer"
      }
    },
    {
      "id": "08f2b16f-766f-4d80-8a16-7b41ce4da472",
      "name": "Sticky Note3",
      "type": "n8n-nodes-base.stickyNote",
      "position": [
        -1200,
        40
      ],
      "parameters": {
        "width": 200,
        "height": 500,
        "content": "## How it works\nThis template shows how to calculate a workflow evaluation metric: **whether an output matches an expected output** (i.e. has the same meaning).\n\nThe workflow takes questions about the"
      }
    },
    {
      "id": "e8674263-6cb6-49dc-9b93-3ce167b35608",
      "name": "Sticky Note4",
      "type": "n8n-nodes-base.stickyNote",
      "position": [
        -960,
        280
      ],
      "parameters": {
        "width": 220,
        "height": 220,
        "content": "Read in [this test dataset](https://docs.google.com/spreadsheets/d/1uuPS5cHtSNZ6HNLOi75A2m8nVWZrdBZ_Ivf58osDAS8/edit?gid=662663849#gid=662663849) of questions"
      }
    },
    {
      "id": "edcd9964-51a1-49bd-8a9e-ebc9b4d0e963",
      "name": "OpenAI Chat Model",
      "type": "@n8n/n8n-nodes-langchain.lmChatOpenAi",
      "position": [
        -440,
        420
      ]
    },
    {
      "id": "f5b9f75a-9a9c-48cf-93a6-16407c730340",
      "name": "When fetching a dataset row",
      "type": "n8n-nodes-base.evaluationTrigger",
      "position": [
        -900,
        340
      ]
    },
    {
      "id": "411fb522-c5d4-4c24-ba0f-cb830e1b63c4",
      "name": "Evaluating?",
      "type": "n8n-nodes-base.evaluation",
      "position": [
        -60,
        200
      ]
    },
    {
      "id": "01b7bd96-00e5-4618-9797-8477b41ad78b",
      "name": "AI Agent",
      "type": "@n8n/n8n-nodes-langchain.agent",
      "position": [
        -440,
        200
      ]
    },
    {
      "id": "886ee0aa-db8a-4b64-a9d6-ac4fc865a36b",
      "name": "Calculate correctness metric",
      "type": "@n8n/n8n-nodes-langchain.openAi",
      "position": [
        220,
        80
      ]
    },
    {
      "id": "6157d456-aa3c-4cca-9d9e-9f5fd19eae68",
      "name": "When chat message received",
      "type": "@n8n/n8n-nodes-langchain.chatTrigger",
      "position": [
        -900,
        100
      ]
    },
    {
      "id": "75aec6a1-376a-489e-940c-4868e8d8bcbb",
      "name": "Match chat format",
      "type": "n8n-nodes-base.set",
      "position": [
        -680,
        340
      ]
    },
    {
      "id": "04548ab1-8644-47d3-9652-4552d798853a",
      "name": "Sticky Note",
      "type": "n8n-nodes-base.stickyNote",
      "position": [
        -80,
        100
      ],
      "parameters": {
        "width": 150,
        "height": 260,
        "content": "Only calculate metrics if we're evaluating, to reduce costs"
      }
    },
    {
      "id": "792ccfd0-387a-46bc-b68b-948fcd2098dd",
      "name": "Return chat response",
      "type": "n8n-nodes-base.noOp",
      "position": [
        220,
        340
      ]
    },
    {
      "id": "1bb9466a-439a-41ff-a425-5550127786d4",
      "name": "Set metrics",
      "type": "n8n-nodes-base.evaluation",
      "position": [
        580,
        80
      ]
    }
  ],
  "connections": {
    "AI Agent": {
      "main": [
        [
          {
            "node": "Evaluating?",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Evaluating?": {
      "main": [
        [
          {
            "node": "Calculate correctness metric",
            "type": "main",
            "index": 0
          }
        ],
        [
          {
            "node": "Return chat response",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Match chat format": {
      "main": [
        [
          {
            "node": "AI Agent",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "OpenAI Chat Model": {
      "ai_languageModel": [
        [
          {
            "node": "AI Agent",
            "type": "ai_languageModel",
            "index": 0
          }
        ]
      ]
    },
    "When chat message received": {
      "main": [
        [
          {
            "node": "AI Agent",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "When fetching a dataset row": {
      "main": [
        [
          {
            "node": "Match chat format",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Calculate correctness metric": {
      "main": [
        [
          {
            "node": "Set metrics",
            "type": "main",
            "index": 0
          }
        ]
      ]
    }
  }
}