{
 "set": "tool-search-v1",
 "updated": "2026-10-03T13:43:18.168Z",
 "months": [
  {
   "month": "2026-10",
   "set": "tool-search-v1",
   "run_at": "2026-10-03T13:43:18.168Z",
   "tools_indexed": 251975,
   "jobs": 133,
   "p_at_1": 0.774,
   "p_at_5": 0.678,
   "ndcg_at_5": 0.689,
   "mrr": 0.863,
   "answering_top5_share": 0.985,
   "no_relevant_top10": 1,
   "median_latency_ms": 697,
   "median_server_ms": 412,
   "url": "https://7it.co.il/7maps/search-eval/results/2026-10.json"
  }
 ],
 "latest": {
  "month": "2026-10",
  "metrics": {
   "jobs": 133,
   "p_at_1": 0.774,
   "p_at_5": 0.678,
   "ndcg_at_5": 0.689,
   "mrr": 0.863,
   "answering_top5_share": 0.985,
   "no_relevant_top10": 1,
   "median_latency_ms": 697,
   "median_server_ms": 412
  },
  "by_category": {
   "email": {
    "jobs": 6,
    "p_at_5": 0.567,
    "ndcg_at_5": 0.608,
    "mrr": 0.774
   },
   "calendar": {
    "jobs": 5,
    "p_at_5": 0.72,
    "ndcg_at_5": 0.731,
    "mrr": 0.85
   },
   "database": {
    "jobs": 7,
    "p_at_5": 0.543,
    "ndcg_at_5": 0.492,
    "mrr": 0.636
   },
   "web search": {
    "jobs": 5,
    "p_at_5": 0.48,
    "ndcg_at_5": 0.536,
    "mrr": 0.8
   },
   "payments": {
    "jobs": 6,
    "p_at_5": 0.733,
    "ndcg_at_5": 0.748,
    "mrr": 0.889
   },
   "files": {
    "jobs": 5,
    "p_at_5": 0.68,
    "ndcg_at_5": 0.752,
    "mrr": 1
   },
   "code repository": {
    "jobs": 6,
    "p_at_5": 0.633,
    "ndcg_at_5": 0.688,
    "mrr": 0.917
   },
   "CRM": {
    "jobs": 5,
    "p_at_5": 0.84,
    "ndcg_at_5": 0.829,
    "mrr": 0.85
   },
   "docs and knowledge": {
    "jobs": 6,
    "p_at_5": 0.433,
    "ndcg_at_5": 0.57,
    "mrr": 1
   },
   "maps and geo": {
    "jobs": 5,
    "p_at_5": 0.8,
    "ndcg_at_5": 0.816,
    "mrr": 1
   },
   "weather": {
    "jobs": 4,
    "p_at_5": 0.7,
    "ndcg_at_5": 0.708,
    "mrr": 0.875
   },
   "translation": {
    "jobs": 4,
    "p_at_5": 0.8,
    "ndcg_at_5": 0.752,
    "mrr": 0.75
   },
   "scraping": {
    "jobs": 5,
    "p_at_5": 0.56,
    "ndcg_at_5": 0.542,
    "mrr": 0.633
   },
   "images": {
    "jobs": 5,
    "p_at_5": 0.64,
    "ndcg_at_5": 0.659,
    "mrr": 0.867
   },
   "analytics": {
    "jobs": 5,
    "p_at_5": 0.64,
    "ndcg_at_5": 0.571,
    "mrr": 0.767
   },
   "ticketing": {
    "jobs": 5,
    "p_at_5": 0.52,
    "ndcg_at_5": 0.549,
    "mrr": 0.733
   },
   "chat": {
    "jobs": 5,
    "p_at_5": 0.76,
    "ndcg_at_5": 0.794,
    "mrr": 1
   },
   "storage": {
    "jobs": 4,
    "p_at_5": 0.8,
    "ndcg_at_5": 0.814,
    "mrr": 1
   },
   "auth and identity": {
    "jobs": 4,
    "p_at_5": 0.8,
    "ndcg_at_5": 0.761,
    "mrr": 0.875
   },
   "monitoring": {
    "jobs": 6,
    "p_at_5": 0.633,
    "ndcg_at_5": 0.471,
    "mrr": 0.639
   },
   "spreadsheets": {
    "jobs": 3,
    "p_at_5": 0.467,
    "ndcg_at_5": 0.582,
    "mrr": 1
   },
   "project management": {
    "jobs": 3,
    "p_at_5": 0.8,
    "ndcg_at_5": 0.823,
    "mrr": 1
   },
   "finance data": {
    "jobs": 3,
    "p_at_5": 0.8,
    "ndcg_at_5": 0.836,
    "mrr": 1
   },
   "crypto": {
    "jobs": 2,
    "p_at_5": 0.9,
    "ndcg_at_5": 0.831,
    "mrr": 0.75
   },
   "audio and video": {
    "jobs": 4,
    "p_at_5": 0.8,
    "ndcg_at_5": 0.846,
    "mrr": 1
   },
   "social media": {
    "jobs": 2,
    "p_at_5": 0.6,
    "ndcg_at_5": 0.67,
    "mrr": 1
   },
   "ecommerce": {
    "jobs": 3,
    "p_at_5": 0.733,
    "ndcg_at_5": 0.738,
    "mrr": 0.833
   },
   "dev infrastructure": {
    "jobs": 5,
    "p_at_5": 0.88,
    "ndcg_at_5": 0.86,
    "mrr": 0.9
   },
   "notes and memory": {
    "jobs": 2,
    "p_at_5": 0.6,
    "ndcg_at_5": 0.67,
    "mrr": 1
   },
   "utilities": {
    "jobs": 3,
    "p_at_5": 0.867,
    "ndcg_at_5": 0.913,
    "mrr": 1
   }
  },
  "comparison": [
   {
    "name": "Glama",
    "site": "https://glama.ai/",
    "checked": "2026-10-03",
    "comparable": false,
    "reason": "Not comparable. The tool search API (https://glama.ai/api/mcp/v1/servers) answers 401 without an API key tied to a Glama account; robots.txt (https://glama.ai/robots.txt) disallows /api/ for all user agents; and the terms (https://glama.ai/policies/terms-of-service, section 15.6) forbid using the API data to benchmark a product that competes with the Glama directory and forbid scraping the website to obtain data the API gates. So 7Maps does not query Glama for this evaluation.",
    "urls": [
     "https://glama.ai/api/mcp/v1/servers",
     "https://glama.ai/robots.txt",
     "https://glama.ai/policies/terms-of-service"
    ]
   }
  ],
  "worst": [
   {
    "id": "web-2",
    "job": "get the top search engine results for a query",
    "ndcg5": 0,
    "rr": 0
   },
   {
    "id": "email-3",
    "job": "read the latest unread emails",
    "ndcg5": 0,
    "rr": 0.143
   },
   {
    "id": "db-2",
    "job": "list the tables in a MySQL database",
    "ndcg5": 0.044,
    "rr": 0.2
   },
   {
    "id": "calendar-3",
    "job": "find a free time slot for three people",
    "ndcg5": 0.146,
    "rr": 0.25
   },
   {
    "id": "monitor-6",
    "job": "list open incidents on PagerDuty",
    "ndcg5": 0.149,
    "rr": 0.333
   },
   {
    "id": "monitor-2",
    "job": "list recent errors from Sentry",
    "ndcg5": 0.164,
    "rr": 0.5
   },
   {
    "id": "scrape-1",
    "job": "fetch a web page and return it as markdown",
    "ndcg5": 0.17,
    "rr": 0.333
   },
   {
    "id": "ticket-2",
    "job": "change the status of a Jira issue",
    "ndcg5": 0.17,
    "rr": 0.333
   },
   {
    "id": "ticket-4",
    "job": "add a comment to a support ticket",
    "ndcg5": 0.17,
    "rr": 0.333
   },
   {
    "id": "email-5",
    "job": "check whether an email address is deliverable",
    "ndcg5": 0.214,
    "rr": 0.5
   }
  ]
 },
 "page": "https://7it.co.il/7maps/search-eval/"
}