# @title Download AgentVista dataset from Hugging Face
import os
from huggingface_hub import snapshot_download
DATASET_DIR = "/content/datasets"
snapshot_download(
repo_id="Warrieryes/AgentVista",
repo_type="dataset",
local_dir=DATASET_DIR,
local_dir_use_symlinks=False,
)
print("Dataset downloaded to:", DATASET_DIR)
print("Top level files:", os.listdir(DATASET_DIR)[:20])
/usr/local/lib/python3.12/dist-packages/huggingface_hub/file_download.py:982: UserWarning: `local_dir_use_symlinks` parameter is deprecated and will be ignored. The process to download files to a local folder has been updated and do not rely on symlinks anymore. You only need to pass a destination folder as`local_dir`.
For more details, check out https://huggingface.co/docs/huggingface_hub/main/en/guides/download#download-files-to-local-folder.
warnings.warn(
Dataset downloaded to: /content/datasets
Top level files: ['.cache', '.gitattributes', 'data', 'README.md']
# @title Enter API keys
import os
from getpass import getpass
# Harness LLM judge (AgentVista accuracy_score): after inference, compares model output to ground truth.
# Required env vars (see AgentVista-main/utils/general_qa_tool.py):
# VERIFIER_API_KEY — same OpenRouter key is fine (set below).
# VERIFIER_END_POINT — OpenRouter chat-completions URL (set below; also set again in the next cell).
# VERIFIER_MODEL_NAME — optional; defaults to gpt-4o in the harness if unset; we set an OpenRouter slug below.
OPENROUTER_CHAT = "https://openrouter.ai/api/v1/chat/completions"
os.environ["REASONING_END_POINT"] = OPENROUTER_CHAT
os.environ["VERIFIER_END_POINT"] = OPENROUTER_CHAT
# Cheap, text-only judge on OpenRouter (change if you prefer e.g. openai/gpt-4.1)
os.environ["VERIFIER_MODEL_NAME"] = os.environ.get("VERIFIER_MODEL_NAME", "gpt-5.4-mini")
os.environ["REASONING_API_KEY"] = getpass("OpenRouter API key: ")
os.environ["SERPAPI_KEY"] = getpass("Serper API key: ")
jina_key = getpass("Jina API key (press Enter to skip): ")
os.environ["JINA_API_KEY"] = jina_key if jina_key.strip() else ""
verifier_key = getpass("Verifier API key via OpenRouter (press Enter to reuse reasoning key): ")
os.environ["VERIFIER_API_KEY"] = verifier_key if verifier_key.strip() else os.environ["REASONING_API_KEY"]
print("Keys set.")
print("Harness judge: VERIFIER_END_POINT =", os.environ["VERIFIER_END_POINT"])
print("Harness judge: VERIFIER_MODEL_NAME =", os.environ["VERIFIER_MODEL_NAME"])
OpenRouter API key: ··········
Serper API key: ··········
Jina API key (press Enter to skip): ··········
Verifier API key via OpenRouter (press Enter to reuse reasoning key): ··········
Keys set.
Harness judge: VERIFIER_END_POINT = https://openrouter.ai/api/v1/chat/completions
Harness judge: VERIFIER_MODEL_NAME = gpt-5.4-mini
# @title Configure OpenRouter for multi model evaluation
import os
OPENROUTER_ENDPOINT = "https://openrouter.ai/api/v1/chat/completions"
MODELS = [
"openai/gpt-5.4",
"qwen/qwen3.5-35b-a3b",
"google/gemini-3.1-pro-preview",
]
os.environ["REASONING_END_POINT"] = OPENROUTER_ENDPOINT
os.environ["VERIFIER_END_POINT"] = OPENROUTER_ENDPOINT
# If you skipped cell 3, still pick a judge model for OpenRouter (text-only scoring).
os.environ.setdefault("VERIFIER_MODEL_NAME", "openai/gpt-5.4-mini")
os.environ["ENABLED_TOOLS"] = "web_search,image_search,visit,code_interpreter"
print("OpenRouter endpoint:", OPENROUTER_ENDPOINT)
print("Harness judge model (VERIFIER_MODEL_NAME):", os.environ.get("VERIFIER_MODEL_NAME"))
print("Enabled tools:", os.environ["ENABLED_TOOLS"])
print("\nModels selected for comparison:")
for m in MODELS:
print(" ", m)
OpenRouter endpoint: https://openrouter.ai/api/v1/chat/completions
Harness judge model (VERIFIER_MODEL_NAME): gpt-5.4-mini
Enabled tools: web_search,image_search,visit,code_interpreter
Models selected for comparison:
openai/gpt-5.4
qwen/qwen3.5-35b-a3b
google/gemini-3.1-pro-preview
# @title Load parquet dataset and export embedded images safely
import os
import io
import json
import pandas as pd
import numpy as np
from glob import glob
from PIL import Image
DATASET_DIR = "/content/datasets"
EXPORT_DIR = os.path.join(DATASET_DIR, "exported_images")
os.makedirs(EXPORT_DIR, exist_ok=True)
parquet_files = glob(os.path.join(DATASET_DIR, "**", "*.parquet"), recursive=True)
if not parquet_files:
raise FileNotFoundError("No parquet files found in dataset directory.")
print("Found parquet files:", parquet_files)
dfs = [pd.read_parquet(p) for p in parquet_files]
df = pd.concat(dfs, ignore_index=True)
print("Total samples loaded:", len(df))
print("Columns:", df.columns.tolist())
def unwrap_object(x):
"""
Recursively unwrap numpy object arrays, singleton tuples, and nested containers.
"""
while True:
if isinstance(x, np.ndarray):
if x.dtype == object:
if x.size == 1:
x = x.item()
continue
return x.tolist()
return x
if isinstance(x, tuple):
if len(x) == 1:
x = x[0]
continue
return list(x)
if isinstance(x, list):
if len(x) == 1 and not isinstance(x[0], (str, bytes, bytearray, dict)):
x = x[0]
continue
return x
return x
def save_one_image(img_obj, out_path_base):
"""
Save one image object to disk and return its absolute path.
Supports bytes, dicts, PIL images, and numeric numpy arrays.
"""
img_obj = unwrap_object(img_obj)
if isinstance(img_obj, (bytes, bytearray)):
img = Image.open(io.BytesIO(img_obj)).convert("RGB")
out_path = f"{out_path_base}.png"
img.save(out_path)
return out_path
if isinstance(img_obj, dict):
if "bytes" in img_obj and img_obj["bytes"] is not None:
img = Image.open(io.BytesIO(img_obj["bytes"])).convert("RGB")
out_path = f"{out_path_base}.png"
img.save(out_path)
return out_path
if "path" in img_obj and img_obj["path"]:
return img_obj["path"]
if isinstance(img_obj, Image.Image):
out_path = f"{out_path_base}.png"
img_obj.convert("RGB").save(out_path)
return out_path
if isinstance(img_obj, np.ndarray):
if img_obj.dtype == object:
raise TypeError(f"Still received object ndarray after unwrap: shape={img_obj.shape}")
img = Image.fromarray(img_obj)
out_path = f"{out_path_base}.png"
img.save(out_path)
return out_path
raise TypeError(f"Unsupported image object type after unwrap: {type(img_obj)}")
def normalize_images(images_field, sample_idx):
"""
Convert the parquet images field into a list of relative image paths.
"""
images_field = unwrap_object(images_field)
if isinstance(images_field, (bytes, bytearray, dict, Image.Image, np.ndarray)):
images_field = [images_field]
if isinstance(images_field, tuple):
images_field = list(images_field)
if not isinstance(images_field, list):
raise TypeError(f"Unexpected images field type: {type(images_field)}")
rel_paths = []
for j, img_obj in enumerate(images_field):
out_base = os.path.join(EXPORT_DIR, f"sample_{sample_idx}_img_{j}")
saved_path = save_one_image(img_obj, out_base)
rel_path = os.path.relpath(saved_path, DATASET_DIR)
rel_paths.append(rel_path)
return rel_paths
print("\nExample domains:")
print(sorted(df["domain"].dropna().unique().tolist()))
Found parquet files: ['/content/datasets/data/train-00000-of-00002.parquet', '/content/datasets/data/train-00001-of-00002.parquet']
Total samples loaded: 209
Columns: ['images', 'problem', 'answer', 'domain', 'subdomain']
Example domains:
['academics', 'commerce', 'culture', 'entertainment', 'geography', 'society', 'technology']
# @title Create a stratified AgentVista subset for side by side model comparison
import os
import json
DOMAINS = [
"commerce",
"entertainment",
"technology",
]
SAMPLES_PER_DOMAIN = 5
STRATIFIED_VAL_PATH = os.path.join(DATASET_DIR, "val_domain_smoke_test.json")
subset = []
sample_counter = 0
for domain in DOMAINS:
domain_df = df[df["domain"] == domain].copy()
if len(domain_df) == 0:
continue
take = min(SAMPLES_PER_DOMAIN, len(domain_df))
sampled = domain_df.sample(n=take, random_state=42)
for _, row in sampled.iterrows():
image_paths = normalize_images(row["images"], sample_counter)
subset.append({
"question_id": f"sample_{sample_counter}",
"question": row["problem"],
"problem": row["problem"],
"images": image_paths,
"solution": row["answer"],
"answer": row["answer"],
"domain": row["domain"],
"subdomain": row["subdomain"],
})
sample_counter += 1
with open(STRATIFIED_VAL_PATH, "w", encoding="utf-8") as f:
json.dump(subset, f, ensure_ascii=False, indent=2)
print("Saved stratified subset to:", STRATIFIED_VAL_PATH)
print("Number of samples:", len(subset))
print("Domains included:", sorted(set(x["domain"] for x in subset)))
print("\nFirst sample preview:")
print(json.dumps(subset[0], indent=2, ensure_ascii=False)[:1500])
Saved stratified subset to: /content/datasets/val_domain_smoke_test.json
Number of samples: 15
Domains included: ['commerce', 'entertainment', 'technology']
First sample preview:
{
"question_id": "sample_0",
"question": "\nIt is now October 2025, and I need to rent office space for a team of 5 employees, and the space must meet the requirements in the local workplace safety regulations regarding the minimum area per employee. Considering the current market rental levels and typical utility costs in this area, which room number has the lowest total monthly cost (rent plus utilities) while meeting the legal space requirements for 5 employees?",
"problem": "\nIt is now October 2025, and I need to rent office space for a team of 5 employees, and the space must meet the requirements in the local workplace safety regulations regarding the minimum area per employee. Considering the current market rental levels and typical utility costs in this area, which room number has the lowest total monthly cost (rent plus utilities) while meeting the legal space requirements for 5 employees?",
"images": [
"exported_images/sample_0_img_0.png"
],
"solution": "Room 28",
"answer": "Room 28",
"domain": "commerce",
"subdomain": "Transaction and Price Calculation"
}
# @title Validate subset structure and exported images
import os
import json
from PIL import Image
import matplotlib.pyplot as plt
with open(STRATIFIED_VAL_PATH, "r", encoding="utf-8") as f:
subset_records = json.load(f)
print("Loaded subset records:", len(subset_records))
required_fields = ["question_id", "question", "images", "answer", "domain", "subdomain"]
for i, rec in enumerate(subset_records[:5]):
missing = [k for k in required_fields if k not in rec]
if missing:
raise ValueError(f"Sample {i} is missing required fields: {missing}")
print("Schema check passed for first 5 samples.")
missing_images = []
bad_images = []
image_summary = []
for rec in subset_records:
qid = rec["question_id"]
for img_rel in rec["images"]:
img_abs = os.path.join(DATASET_DIR, img_rel)
if not os.path.exists(img_abs):
missing_images.append((qid, img_rel))
continue
try:
with Image.open(img_abs) as img:
width, height = img.size
image_summary.append({
"question_id": qid,
"image": img_rel,
"format": img.format,
"mode": img.mode,
"width": width,
"height": height,
})
except Exception as e:
bad_images.append((qid, img_rel, str(e)))
print("Total exported images checked:", len(image_summary))
print("Missing images:", len(missing_images))
print("Unreadable images:", len(bad_images))
if missing_images[:5]:
print("\nExample missing images:")
for x in missing_images[:5]:
print(x)
if bad_images[:5]:
print("\nExample unreadable images:")
for x in bad_images[:5]:
print(x)
if missing_images or bad_images:
raise ValueError("Image validation failed. Fix paths or exports before continuing.")
print("\nImage validation passed.")
# Show a few examples visually
n_show = min(4, len(subset_records))
fig, axes = plt.subplots(n_show, 1, figsize=(10, 4 * n_show))
if n_show == 1:
axes = [axes]
for ax, rec in zip(axes, subset_records[:n_show]):
first_img = rec["images"][0]
img_path = os.path.join(DATASET_DIR, first_img)
img = Image.open(img_path)
ax.imshow(img)
ax.set_title(f'{rec["question_id"]} | {rec["domain"]} | {rec["subdomain"]}\n{rec["question"][:120]}')
ax.axis("off")
plt.tight_layout()
plt.show()
Loaded subset records: 15
Schema check passed for first 5 samples.
Total exported images checked: 18
Missing images: 0
Unreadable images: 0
Image validation passed.
# @title Tiny sanity check on 2 samples with one model before the full run
import os
import json
import subprocess
SANITY_PATH = os.path.join(DATASET_DIR, "val_sanity_check.json")
with open(STRATIFIED_VAL_PATH, "r", encoding="utf-8") as f:
full_subset = json.load(f)
sanity_subset = full_subset[:2]
with open(SANITY_PATH, "w", encoding="utf-8") as f:
json.dump(sanity_subset, f, ensure_ascii=False, indent=2)
sanity_model = MODELS[0]
sanity_output_dir = "/content/agentvista_sanity_run"
os.makedirs(sanity_output_dir, exist_ok=True)
env = os.environ.copy()
env["REASONING_MODEL_NAME"] = sanity_model
env["VERIFIER_MODEL_NAME"] = sanity_model
cmd = [
"python", "infer.py",
"--input-file", SANITY_PATH,
"--image-folder", DATASET_DIR,
"--output-dir", sanity_output_dir,
"--max-turns", "8",
"--max-images", "10",
"--max-total-tokens", "16000",
"--skip-completed",
]
print("Running sanity check model:", sanity_model)
result = subprocess.run(cmd, env=env, text=True, capture_output=True)
print(result.stdout[-4000:])
if result.returncode != 0:
print("\nError output:")
print(result.stderr[-4000:])
Running sanity check model: openai/gpt-5.4
ck
[Visit] Fallback to Jina API after other methods failed...
[Visit] Jina API request failed (attempt 1/3): HTTPSConnectionPool(host='r.jina.ai', port=443): Read timed out. (read timeout=20)
[Visit] Jina API request failed (attempt 2/3): HTTPSConnectionPool(host='r.jina.ai', port=443): Read timed out. (read timeout=20)
[Visit] Jina API request failed (attempt 3/3): HTTPSConnectionPool(host='r.jina.ai', port=443): Read timed out. (read timeout=20)
[Image Processing] Skipping image extraction for visit (returns text summary, images should not be extracted)
[API] No fallback API configured, using primary API with 2 retries
[Primary API] API Request attempt 1:
Model: openai/gpt-5.4
Messages count: 10
Temperature: 0.0
Message 0: role=system, content_preview=You are a visual reasoning agent. Your goal is to answer questions about images.
# AVAILABLE TOOLS:...
Message 1: role=user, content_preview=[{'type': 'image_url', 'image_url': {'url': 'data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/2wBD...
Message 2: role=assistant, content_preview=I’ll first extract the relevant room areas from the floor plan and identify which rooms could legall...
... and 7 more messages
[Reasoning] Using content field as reasoning_text: 1679 chars
[Tool Call] web_search with params: {'query': 'Тольятти офис коммунальные услуги в среднем 5000 руб 20 м2 октябрь 2025 аренда офис 20 м2 12000 Юбилейная 31', 'max_results': 10}
[Tool Cache] Reusing existing instance of web_search
[WebSearch] Searching for: Тольятти офис коммунальные услуги в среднем 5000 руб 20 м2 октябрь 2025 аренда офис 20 м2 12000 Юбилейная 31 (attempt 1/3)
[WebSearch] Found 10 results
[API] No fallback API configured, using primary API with 2 retries
[Primary API] API Request attempt 1:
Model: openai/gpt-5.4
Messages count: 12
Temperature: 0.0
Message 0: role=system, content_preview=You are a visual reasoning agent. Your goal is to answer questions about images.
# AVAILABLE TOOLS:...
Message 1: role=user, content_preview=[{'type': 'image_url', 'image_url': {'url': 'data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/2wBD...
Message 2: role=assistant, content_preview=I’ll first extract the relevant room areas from the floor plan and identify which rooms could legall...
... and 9 more messages
[API] No fallback API configured, using primary API with 2 retries
[Primary API] API Request attempt 1:
Model: openai/gpt-5.4
Messages count: 2
Temperature: 0.0
Message 0: role=system, content_preview=You are a visual reasoning agent. Your goal is to answer questions about images.
# AVAILABLE TOOLS:...
Message 1: role=user, content_preview=[{'type': 'image_url', 'image_url': {'url': 'data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/2wBD...
[Reasoning] Using content field as reasoning_text: 817 chars
[Tool Call] web_search with params: {'query': '2025-12-01 HKD to CNY exchange rate 1 HKD CNY December 1 2025', 'max_results': 5}
[Tool Cache] Created new instance of web_search
[WebSearch] Searching for: 2025-12-01 HKD to CNY exchange rate 1 HKD CNY December 1 2025 (attempt 1/3)
[WebSearch] Found 5 results
[API] No fallback API configured, using primary API with 2 retries
[Primary API] API Request attempt 1:
Model: openai/gpt-5.4
Messages count: 4
Temperature: 0.0
Message 0: role=system, content_preview=You are a visual reasoning agent. Your goal is to answer questions about images.
# AVAILABLE TOOLS:...
Message 1: role=user, content_preview=[{'type': 'image_url', 'image_url': {'url': 'data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/2wBD...
Message 2: role=assistant, content_preview=I need identify the circled order details and total HKD, then convert using HKD→RMB exchange rate on...
... and 1 more messages
Results saved to /content/agentvista_sanity_run/results.jsonl
--- Evaluation Summary ---
Total samples: 2
Overall accuracy: 0.5000
Average turns: 4.00
Summary saved to /content/agentvista_sanity_run/summary_metrics.json
---
# @title Inspect sanity check predictions before full evaluation
import os
import json
import pandas as pd
sanity_results_path = "/content/agentvista_sanity_run/results.jsonl"
sanity_rows = []
if os.path.exists(sanity_results_path):
with open(sanity_results_path, "r", encoding="utf-8") as f:
for line in f:
sanity_rows.append(json.loads(line))
sanity_df = pd.DataFrame(sanity_rows)
print("Sanity result columns:", sanity_df.columns.tolist())
cols = [c for c in ["question_id", "final_answer", "ground_truth", "reasoning_text"] if c in sanity_df.columns]
display(sanity_df[cols].head(10))
# @title Inspect sanity predictions in detail
import pandas as pd
import json
import os
sanity_results_path = "/content/agentvista_sanity_run/results.jsonl"
rows = []
with open(sanity_results_path, "r", encoding="utf-8") as f:
for line in f:
rows.append(json.loads(line))
sanity_df = pd.DataFrame(rows)
display(sanity_df[["question_id", "prompt", "final_answer", "ground_truth", "accuracy_score"]])
question_id prompt \
0 sample_0 It is now October 2025, and I need to rent off...
1 sample_1 This is a photo I took on December 1st, 2025, ...
final_answer ground_truth \
0 Room **28** is the cheapest compliant choice.\... Room 28
1 你一共支付了 **約 54.49 元人民幣**（60 港幣 × 2025年12月1日匯率 1... 54.6
accuracy_score
0 1.0
1 0.0
# @title Run AgentVista on the same subset for all selected models
import os
import subprocess
INPUT_FILE = STRATIFIED_VAL_PATH
IMAGE_FOLDER = DATASET_DIR
BASE_OUTPUT_DIR = "/content/agentvista_multi_model_runs"
os.makedirs(BASE_OUTPUT_DIR, exist_ok=True)
for model_name in MODELS:
safe_name = model_name.replace("/", "__")
output_dir = os.path.join(BASE_OUTPUT_DIR, safe_name)
os.makedirs(output_dir, exist_ok=True)
env = os.environ.copy()
env["REASONING_MODEL_NAME"] = model_name
env["VERIFIER_MODEL_NAME"] = model_name
cmd = [
"python", "infer.py",
"--input-file", INPUT_FILE,
"--image-folder", IMAGE_FOLDER,
"--output-dir", output_dir,
"--max-turns", "10",
"--max-images", "20",
"--max-total-tokens", "24000",
"--skip-completed",
]
print("\nRunning model:", model_name)
print("Output directory:", output_dir)
result = subprocess.run(cmd, env=env, text=True, capture_output=True)
print(result.stdout[-3000:])
if result.returncode != 0:
print("\nError output:")
print(result.stderr[-3000:])
Running model: openai/gpt-5.4
Output directory: /content/agentvista_multi_model_runs/openai__gpt-5.4
he chart (likely dots.vlm1) and identify benchmark comparison details for OCR, Document, and Chart tasks versus Gemini 2.5 Pro; specifically which benchmark names show the open-source model outperforming Gemini 2.5 Pro.'}
[Tool Cache] Created new instance of visit
[Visit] Fetching URL: https://huggingface.co/spaces/opencompass/open_vlm_leaderboard
[Visit] Goal: Find the top open-source model in the chart (likely dots.vlm1) and identify benchmark comparison details for OCR, Document, and Chart tasks versus Gemini 2.5 Pro; specifically which benchmark names show the open-source model outperforming Gemini 2.5 Pro.
[Visit] Trying Jina API first...
[Visit] Jina API extraction successful: 1044 characters
[Visit] Extracted 1044 characters
[Visit] Calling API to summarize content (model: openai/gpt-5.4)...
[Visit] API summarization successful (evidence: 661 chars, summary: 361 chars)
[Image Processing] Skipping image extraction for visit (returns text summary, images should not be extracted)
[API] No fallback API configured, using primary API with 2 retries
[Primary API] API Request attempt 1:
Model: openai/gpt-5.4
Messages count: 11
Temperature: 0.0
Message 0: role=system, content_preview=You are a visual reasoning agent. Your goal is to answer questions about images.
# AVAILABLE TOOLS:...
Message 1: role=user, content_preview=[{'type': 'image_url', 'image_url': {'url': 'data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/2wBD...
Message 2: role=assistant, content_preview=I first inspect the provided image. It is a scatter plot of model performance over time, with blue p...
... and 8 more messages
[Reasoning] Using content field as reasoning_text: 686 chars
[Tool Call] web_search with params: {'query': 'dots.vlm1 Gemini 2.5 Pro OCR Document Chart benchmark comparison dots.vlm1 benchmark names', 'max_results': 10}
[Tool Cache] Reusing existing instance of web_search
[WebSearch] Searching for: dots.vlm1 Gemini 2.5 Pro OCR Document Chart benchmark comparison dots.vlm1 benchmark names (attempt 1/3)
[WebSearch] Found 10 results
[API] No fallback API configured, using primary API with 2 retries
[Primary API] API Request attempt 1:
Model: openai/gpt-5.4
Messages count: 13
Temperature: 0.0
Message 0: role=system, content_preview=You are a visual reasoning agent. Your goal is to answer questions about images.
# AVAILABLE TOOLS:...
Message 1: role=user, content_preview=[{'type': 'image_url', 'image_url': {'url': 'data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/2wBD...
Message 2: role=assistant, content_preview=I first inspect the provided image. It is a scatter plot of model performance over time, with blue p...
... and 10 more messages
Results saved to /content/agentvista_multi_model_runs/openai__gpt-5.4/results.jsonl
--- Evaluation Summary ---
Total samples: 15
Overall accuracy: 0.1333
Average turns: 2.80
Summary saved to /content/agentvista_multi_model_runs/openai__gpt-5.4/summary_metrics.json
------------------------
Inference finished.
Running model: qwen/qwen3.5-35b-a3b
Output directory: /content/agentvista_multi_model_runs/qwen__qwen3.5-35b-a3b
tVQA"'}
[Tool Cache] Reusing existing instance of web_search
[WebSearch] Searching for: "MiMo-VL 7B-RL" "Gemini 2.5 Pro" "DocVQA" "ChartQA" "TextVQA" (attempt 1/3)
[API] No fallback API configured, using primary API with 2 retries
[Primary API] API Request attempt 1:
Model: qwen/qwen3.5-35b-a3b
Messages count: 16
Temperature: 0.0
Message 0: role=system, content_preview=You are a visual reasoning agent. Your goal is to answer questions about images.
# AVAILABLE TOOLS:...
Message 1: role=user, content_preview=[{'type': 'image_url', 'image_url': {'url': 'data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/2wBD...
Message 2: role=assistant, content_preview=
... and 13 more messages
[Reasoning] Preserved reasoning_details in tool call message
[Tool Call] web_search with
# @title Load and merge outputs from all models
import os
import json
import pandas as pd
all_rows = []
for model_name in MODELS:
safe_name = model_name.replace("/", "__")
results_path = os.path.join(BASE_OUTPUT_DIR, safe_name, "results.jsonl")
if not os.path.exists(results_path):
print("Missing results for:", model_name)
continue
with open(results_path, "r", encoding="utf-8") as f:
for line in f:
row = json.loads(line)
row["model"] = model_name
all_rows.append(row)
results_df = pd.DataFrame(all_rows)
print("Loaded rows:", len(results_df))
print("Columns:", results_df.columns.tolist())
display(results_df.head(10))
question_id prompt \
0 sample_0 It is now October 2025, and I need to rent off...
1 sample_1 This is a photo I took on December 1st, 2025, ...
2 sample_2 I bought the light blue suitcase shown in the ...
3 sample_3 I'm hosting a Halloween party and need red win...
4 sample_4 I took this photo at a really cool pop-up stor...
5 sample_5 How many individual shooting zone units are th...
6 sample_6 Please observe this group of images, whose bac...
7 sample_7 I've been tracking this player count graph bec...
8 sample_8 I want to know the best finishing position tha...
9 sample_9 Please tell me the names of the current first-...
final_answer \
0 Assuming the standard office workplace rule st...
1 你一共付了 **60港币**。 \n按 **2025年12月1日** 约 **1 港币 =...
2 You would most likely need to pay **210 RMB** ...
3 The wine that best matches your criteria is **...
4 The pop-up appears to be the **MINISO x Disney...
5 31
6 1, 3, 4, 6
7 The game is **Monster Hunter Wilds**.\n\nUsing...
8 2nd place
9 I can’t tell the swimmers’ names from this ima...
ground_truth \
0 Room 28
1 54.6
2 690RMB
3 Традиции Абхазии (Traditsii Abkhazii)， 6.82
4 29 days (November 29th to December 28th, 2025)
5 58
6 2, 3, 6
7 Monster Hunter Wilds; 59.86%
8 3
9 First-place: Ryan Held\nLast-place: Drew Kibler
conversation_history accuracy_score \
0 [{'role': 'system', 'content': 'You are a visu... 0.0
1 [{'role': 'system', 'content': 'You are a visu... 1.0
2 [{'role': 'system', 'content': 'You are a visu... 0.0
3 [{'role': 'system', 'content': 'You are a visu... 0.0
4 [{'role': 'system', 'content': 'You are a visu... 0.0
5 [{'role': 'system', 'content': 'You are a visu... 0.0
6 [{'role': 'system', 'content': 'You are a visu... 0.0
7 [{'role': 'system', 'content': 'You are a visu... 0.0
8 [{'role': 'system', 'content': 'You are a visu... 0.0
9 [{'role': 'system', 'content': 'You are a visu... 0.0
trajectory_text trajectory_score \
0 **system**: You are a visual reasoning agent. ... 0.0
1 **system**: You are a visual reasoning agent. ... 1.0
2 **system**: You are a visual reasoning agent. ... 0.0
3 **system**: You are a visual reasoning agent. ... 0.0
4 **system**: You are a visual reasoning agent. ... 0.0
5 **system**: You are a visual reasoning agent. ... 0.0
6 **system**: You are a visual reasoning agent. ... 0.0
7 **system**: You are a visual reasoning agent. ... 0.0
8 **system**: You are a visual reasoning agent. ... 0.0
9 **system**: You are a visual reasoning agent. ... 0.0
trajectory_analysis model
0 the ground truth answer is room 28, but the mo... openai/gpt-5.4
1 the predicted answer computes a total of about... openai/gpt-5.4
2 the ground truth is 690 rmb, but the model pre... openai/gpt-5.4
3 the predicted answer identifies the correct wi... openai/gpt-5.4
4 the predicted answer matches the start and end... openai/gpt-5.4
5 the
# @title Standardize result fields, recover domain, and compute correctness
import os
import json
import re
import pandas as pd
def find_first_existing(df, candidates):
for c in candidates:
if c in df.columns:
return c
return None
pred_col = find_first_existing(results_df, [
"prediction", "pred", "model_answer", "final_answer", "response"
])
gold_col = find_first_existing(results_df, [
"solution", "answer", "ground_truth", "gold_answer"
])
qid_col = find_first_existing(results_df, [
"question_id", "doc_id", "id"
])
domain_col = find_first_existing(results_df, [
"domain"
])
# Harness column names that may hold LLM-judge scores (AgentVista: accuracy_score).
# Populated only when VERIFIER_API_KEY and VERIFIER_END_POINT are set (VERIFIER_MODEL_NAME
# selects the judge on OpenRouter; see cells 3–4). See AgentVista-main/README.md.
# Without API key + endpoint, harness accuracy_score stays 0 for every row.
score_col = find_first_existing(results_df, [
"accuracy_score", "score", "is_correct", "correct", "accuracy"
])
verifier_configured = bool(
os.environ.get("VERIFIER_API_KEY") and os.environ.get("VERIFIER_END_POINT")
)
print("Prediction column:", pred_col)
print("Gold column:", gold_col)
print("Question id column:", qid_col)
print("Domain column from results:", domain_col)
print("Score column:", score_col)
if pred_col is None:
raise ValueError("Could not find a prediction column in results.jsonl.")
if gold_col is None:
raise ValueError("Could not find a ground truth column in results.jsonl.")
if qid_col is None:
raise ValueError("Could not find a question id column in results.jsonl.")
def normalize_text(x):
if x is None:
return ""
x = str(x).strip().lower()
x = re.sub(r"\s+", " ", x)
return x
with open(STRATIFIED_VAL_PATH, "r", encoding="utf-8") as f:
subset_records = json.load(f)
subset_meta = pd.DataFrame(subset_records)[["question_id", "domain", "subdomain", "question"]]
if domain_col is None:
results_df = results_df.merge(
subset_meta,
left_on=qid_col,
right_on="question_id",
how="left"
)
domain_col = "domain"
results_df["pred_norm"] = results_df[pred_col].apply(normalize_text)
results_df["gold_norm"] = results_df[gold_col].apply(normalize_text)
results_df["accuracy_binary_exact"] = (
results_df["pred_norm"] == results_df["gold_norm"]
).astype(float)
if verifier_configured and score_col is not None:
results_df["accuracy_binary"] = results_df[score_col].astype(float)
print(
"Using harness accuracy_score (verifier configured). "
"Column accuracy_binary_exact is normalized string exact match for comparison."
)
else:
results_df["accuracy_binary"] = results_df["accuracy_binary_exact"]
if score_col is not None and not verifier_configured:
print(
"Verifier env vars not set; accuracy_binary uses normalized exact match "
"(raw harness accuracy_score is 0 without VERIFIER_API_KEY / VERIFIER_END_POINT)."
)
# Optional relaxed diagnostic metric
results_df["contains_match"] = results_df.apply(
lambda r: float(
r["gold_norm"] in r["pred_norm"] or r["pred_norm"] in r["gold_norm"]
if r["pred_norm"] and r["gold_norm"] else 0.0
),
axis=1
)
print("\nColumns after recovery:")
print(results_df.columns.tolist())
display(
results_df[
[
c
for c in [
qid_col,
"model",
domain_col,
pred_col,
gold_col,
"accuracy_binary",
"accuracy_binary_exact",
"contains_match",
]
if c in results_df.columns
]
].head(20)
)
Prediction column: final_answer
Gold column: ground_truth
Question id column: question_id
Domain column from results: None
Score column: accuracy_score
Using harness accuracy_score (verifier configured). Column accuracy_binary_exact is normalized string exact match for comparison.
Columns after recovery:
['question_id', 'prompt', 'final_answer', 'ground_truth', 'conversation_history', 'accuracy_score', 'trajectory_text', 'trajectory_score', 'trajectory_analysis', 'model', 'domain', 'subdomain', 'question', 'pred_norm', 'gold_norm', 'accuracy_binary_exact', 'accuracy_binary', 'contains_match']
question_id model domain \
0 sample_0 openai/gpt-5.4 commerce
1 sample_1 openai/gpt-5.4 commerce
2 sample_2 openai/gpt-5.4 commerce
3 sample_3 openai/gpt-5.4 commerce
4 sample_4 openai/gpt-5.4 commerce
5 sample_5 openai/gpt-5.4 entertainment
6 sample_6 openai/gpt-5.4 entertainment
7 sample_7 openai/gpt-5.4 entertainment
8 sample_8 openai/gpt-5.4 entertainment
9 sample_9 openai/gpt-5.4 entertainment
10 sample_10 openai/gpt-5.4 technology
11 sample_11 openai/gpt-5.4 technology
12 sample_12 openai/gpt-5.4 technology
13 sample_13 openai/gpt-5.4 technology
14 sample_14 openai/gpt-5.4 technology
15 sample_0 qwen/qwen3.5-35b-a3b commerce
16 sample_1 qwen/qwen3.5-35b-a3b commerce
17 sample_2 qwen/qwen3.5-35b-a3b commerce
18 sample_3 qwen/qwen3.5-35b-a3b commerce
19 sample_4 qwen/qwen3.5-35b-a3b commerce
final_answer \
0 Assuming the standard office workplace rule st...
1 你一共付了 **60港币**。 \n按 **2025年12月1日** 约 **1 港币 =...
2 You would most likely need to pay **210 RMB** ...
3 The wine that best matches your criteria is **...
4 The pop-up appears to be the **MINISO x Disney...
5 31
6 1, 3, 4, 6
7 The game is **Monster Hunter Wilds**.\n\nUsing...
8 2nd place
9 I can’t tell the swimmers’ names from this ima...
10 **llava-v1.6-vicuna-7b** has the highest Arena...
11 The two wear modes visible are:\n\n1. **Smeari...
12 With the No. 2 and No. 3 gray gears mounted in...
13 11
14 The provided scatter plot shows that the top-p...
15 Room 32
16 Error: All API attempts failed (primary API only)
17 280 RMB
18 Traditions of Abkhazia (Cabernet Sukhumskoe), ...
19 Error: All API attempts failed (primary API only)
ground_truth accuracy_binary \
0 Room 28 0.0
1 54.6 1.0
2 690RMB 0.0
3 Традиции Абхазии (Traditsii Abkhazii)， 6.82 0.0
4 29 days (November 29th to December 28th, 2025) 0.0
5 58 0.0
6 2, 3, 6 0.0
7 Monster Hunter Wilds; 59.86% 0.0
8 3 0.0
9 First-place: Ryan Held\nLast-place: Drew Kibler 0.0
10 Qwen2‑VL‑7B‑Instruct 0.0
11 Micro-vibration and adhesive wear; they degrad... 0.0
12 1, 3, 2, 4, R 1.0
13 23 (or beteen 22 and 24) 0.0
14 charxiv(dq), DOCVQA, ChartQA 0.0
15 Room 28 0.0
16 54.6 0.0
17 690RMB 0.0
18 Традиции Абхазии (Traditsii Abkhazii)， 6.82 0.0
19 29 days (November 29th to December 28th, 2025) 0.0
accuracy_binary_exact contains_match
0 0.0 0.0
1
# @title Build a side by side comparison table
compare_cols = [c for c in [qid_col, domain_col, gold_col, pred_col] if c in results_df.columns]
sample_view = results_df[["model", "accuracy_binary"] + compare_cols].copy()
pivot_values = ["accuracy_binary"]
if pred_col is not None:
pivot_values.append(pred_col)
wide_df = sample_view.pivot_table(
index=[c for c in [qid_col, domain_col, gold_col] if c in sample_view.columns],
columns="model",
values=pivot_values,
aggfunc="first"
).reset_index()
display(wide_df.head(20))
question_id domain \
model
0 sample_0 commerce
1 sample_1 commerce
2 sample_10 technology
3 sample_11 technology
4 sample_12 technology
5 sample_13 technology
6 sample_14 technology
7 sample_2 commerce
8 sample_3 commerce
9 sample_4 commerce
10 sample_5 entertainment
11 sample_6 entertainment
12 sample_7 entertainment
13 sample_8 entertainment
14 sample_9 entertainment
ground_truth \
model
0 Room 28
1 54.6
2 Qwen2‑VL‑7B‑Instruct
3 Micro-vibration and adhesive wear; they degrad...
4 1, 3, 2, 4, R
5 23 (or beteen 22 and 24)
6 charxiv(dq), DOCVQA, ChartQA
7 690RMB
8 Традиции Абхазии (Traditsii Abkhazii)， 6.82
9 29 days (November 29th to December 28th, 2025)
10 58
11 2, 3, 6
12 Monster Hunter Wilds; 59.86%
13 3
14 First-place: Ryan Held\nLast-place: Drew Kibler
accuracy_binary \
model google/gemini-3.1-pro-preview openai/gpt-5.4 qwen/qwen3.5-35b-a3b
0 0.0 0.0 0.0
1 0.0 1.0 0.0
2 0.0 0.0 0.0
3 0.0 0.0 0.0
4 0.0 1.0 1.0
5 0.0 0.0 0.0
6 0.0 0.0 0.0
7 0.0 0.0 0.0
8 0.0 0.0 0.0
9 0.0 0.0 0.0
10 0.0 0.0 0.0
11 0.0 0.0 0.0
12 0.0 0.0 0.0
13 0.0 0.0 0.0
14 0.0 0.0 0.0
final_answer \
model google/gemini-3.1-pro-preview
0 Error: Reached max turns without a definitive ...
1 54.48
2 llava-v1.6-vicuna-7b
3 Error: Reached max turns without a definitive ...
4 Error: Reached max turns without a definitive ...
5 1
6 tags.\nDone.\nI will just write the thought pr...
7 Error: Reached max turns without a definitive ...
8 The wine that meets your criteria with the hig...
9 30
10 Error: Could not parse model response.
11 3, 4, 5, 6
12 Error: Reached max turns without a definitive ...
13 Error: Reached max turns without a definitive ...
14 Error: Could not parse model response.
\
model
# @title Plot per domain accuracy by model
import matplotlib.pyplot as plt
domain_summary = (
results_df.groupby(["model", domain_col], as_index=False)["accuracy_binary"]
.mean()
)
pivot_domain = domain_summary.pivot(index=domain_col, columns="model", values="accuracy_binary")
display(pivot_domain)
ax = pivot_domain.plot(kind="bar", figsize=(12, 6))
ax.set_title("AgentVista mini evaluation by domain")
ax.set_xlabel("Domain")
ax.set_ylabel("Exact match accuracy")
ax.legend(title="Model")
plt.xticks(rotation=45, ha="right")
plt.tight_layout()
plt.show()
model google/gemini-3.1-pro-preview openai/gpt-5.4 \
domain
commerce 0.0 0.2
entertainment 0.0 0.0
technology 0.0 0.2
model qwen/qwen3.5-35b-a3b
domain
commerce 0.0
entertainment 0.0
technology 0.2
# @title Plot overall accuracy by model
import matplotlib.pyplot as plt
overall = (
results_df.groupby("model", as_index=False)["accuracy_binary"]
.mean()
.sort_values("accuracy_binary", ascending=False)
)
display(overall)
plt.figure(figsize=(8, 5))
plt.bar(overall["model"], overall["accuracy_binary"])
plt.title("Overall exact match accuracy on the AgentVista mini subset")
plt.xlabel("Model")
plt.ylabel("Accuracy")
plt.xticks(rotation=25, ha="right")
plt.tight_layout()
plt.show()
# @title Show how to extend the schema with your own use case data
import json
custom_example = [
{
"question_id": "custom_001",
"question": "Look at the screenshot and identify whether the shown product matches the requested specification. Then verify the latest price online and answer with the exact product name and price.",
"images": ["custom_images/product_case_1.png"],
"solution": "Example Ground Truth",
"answer": "Example Ground Truth",
"domain": "commerce",
"subdomain": "product_matching"
},
{
"question_id": "custom_002",
"question": "Inspect the dashboard screenshot, identify the failed component, and verify the vendor documentation for the correct remediation step.",
"images": ["custom_images/ops_case_1.png"],
"solution": "Restart the ingestion connector and revalidate the schema mapping",
"answer": "Restart the ingestion connector and revalidate the schema mapping",
"domain": "technology",
"subdomain": "troubleshooting"
}
]
print(json.dumps(custom_example, indent=2, ensure_ascii=False))
[
{
"question_id": "custom_001",
"question": "Look at the screenshot and identify whether the shown product matches the requested specification. Then verify the latest price online and answer with the exact product name and price.",
"images": [
"custom_images/product_case_1.png"
],
"solution": "Example Ground Truth",
"answer": "Example Ground Truth",
"domain": "commerce",
"subdomain": "product_matching"
},
{
"question_id": "custom_002",
"question": "Inspect the dashboard screenshot, identify the failed component, and verify the vendor documentation for the correct remediation step.",
"images": [
"custom_images/ops_case_1.png"
],
"solution": "Restart the ingestion connector and revalidate the schema mapping",
"answer": "Restart the ingestion connector and revalidate the schema mapping",
"domain": "technology",
"subdomain": "troubleshooting"
}
]
# Store your secrets in a local .env file or your notebook environment.
# Do not hardcode credentials directly in the notebook.
import os
from dotenv import load_dotenv
load_dotenv()
# Get keys for your project from the project settings page: https://cloud.langfuse.com
LANGFUSE_PUBLIC_KEY = os.getenv("LANGFUSE_PUBLIC_KEY")
LANGFUSE_SECRET_KEY = os.getenv("LANGFUSE_SECRET_KEY")
os.environ["LANGFUSE_BASE_URL"] = os.getenv("LANGFUSE_BASE_URL", "https://cloud.langfuse.com")
if not LANGFUSE_PUBLIC_KEY or not LANGFUSE_SECRET_KEY:
raise ValueError(
"Set LANGFUSE_PUBLIC_KEY and LANGFUSE_SECRET_KEY in your environment before running this notebook."
)
os.environ["LANGFUSE_PUBLIC_KEY"] = LANGFUSE_PUBLIC_KEY
os.environ["LANGFUSE_SECRET_KEY"] = LANGFUSE_SECRET_KEY
from dotenv import load_dotenv
import os
load_dotenv()
OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")
if not OPENAI_API_KEY:
print("OPENAI_API_KEY not found.")
OPENAI_API_KEY = input("Enter OPENAI_API_KEY: ").strip()
os.environ["OPENAI_API_KEY"] = OPENAI_API_KEY
print("OPENAI_API_KEY is set.")
print("Langfuse credentials are set.")
OPENAI_API_KEY is set.
Langfuse credentials are set.
from copy import deepcopy
from langfuse.openai import openai
def infer_issue_type(case):
text = case.get("customer_request", "").lower()
if "refund" in text or "charged" in text or "invoice" in text or "subscription" in text:
return "billing_issue"
if "ship" in text or "arrive" in text or "delivery" in text:
return "shipping_issue"
if "log in" in text or "password" in text or "access" in text:
return "account_access_issue"
return "general_support"
def infer_task_definition(case):
return (
f"Resolve this {case['workflow']} support request with the required steps, "
f"correct handoff ({case['expected_handoff']}), and a clear final answer."
)
def infer_expected_plan(case):
plan = [case["required_step"]]
if case["expected_handoff"] != "none":
plan.append(f"handoff_to_{case['expected_handoff']}")
plan.append("respond_to_customer")
return plan
def infer_expected_tools(case):
tools = [
{
"name": case["required_step"],
"description": f"Core {case['workflow']} lookup step.",
"input": {"case_id": case["case_id"], "workflow": case["workflow"]},
"output": {},
}
]
if case["expected_handoff"] != "none":
tools.append(
{
"name": "create_handoff",
"description": "Escalation handoff tool.",
"input": {"target": case["expected_handoff"], "reason": case["workflow"]},
"output": {},
}
)
return tools
def enrich_tool_call(case, call):
return {
"name": str(call.get("name", "unknown_tool")),
"status": str(call.get("status", "unknown")),
"input": call.get("input", {"case_id": case["case_id"], "request": case["customer_request"]}),
"output": call.get("output", {"status": str(call.get("status", "unknown"))}),
"description": call.get("description", f"Support tool in {case['workflow']} workflow."),
}
base_cases = [
{
"case_id": "refund_double_charge",
"workflow": "billing",
"agent_type": "triage_agent",
"customer_request": "I was charged twice for order 8129. Please fix it.",
"required_step": "lookup_order",
"expected_handoff": "billing_specialist",
"reference_answer": "I found the duplicate charge and handed this to billing for refund review.",
"max_retries": 2,
"simulated_retry_count": 1,
"simulated_handoff_target": "billing_specialist",
"simulated_steps_completed": ["lookup_order", "verify_duplicate_charge", "handoff_to_billing_specialist", "respond_to_customer"],
"simulated_tool_calls": [
{"name": "lookup_order", "status": "success", "input": {"order_id": "8129"}},
{"name": "create_handoff", "status": "success", "input": {"target": "billing_specialist"}},
],
"simulated_final_answer": "I confirmed the duplicate charge for order 8129 and escalated this to billing for refund processing. You will receive an update within one business day.",
},
{
"case_id": "shipping_status_no_handoff_needed",
"workflow": "shipping",
"agent_type": "triage_agent",
"customer_request": "Where is my order 9912 and when should it arrive?",
"required_step": "lookup_shipment",
"expected_handoff": "none",
"reference_answer": "Order 9912 is in transit and should arrive tomorrow.",
"max_retries": 1,
"simulated_retry_count": 0,
"simulated_handoff_target": "none",
"simulated_steps_completed": ["lookup_shipment", "respond_to_customer"],
"simulated_tool_calls": [
{"name": "lookup_shipment", "status": "success", "input": {"order_id": "9912"}},
],
"simulated_final_answer": "I checked order 9912 and it is in transit. Carrier ETA is tomorrow.",
},
{
"case_id": "reset_password_wrong_handoff",
"workflow": "account_access",
"agent_type": "triage_agent",
"customer_request": "I cannot log in after changing my password.",
"required_step": "lookup_account_status",
"expected_handoff": "account_security",
"reference_answer": "I checked your account and routed this to account security to restore access.",
"max_retries": 1,
"simulated_retry_count": 0,
"simulated_handoff_target": "billing_specialist",
"simulated_steps_completed": ["lookup_account_status", "handoff_to_billing_specialist", "respond_to_customer"],
"simulated_tool_calls": [
{"name": "lookup_account_status", "status": "success", "input": {"user_id": "user_42"}},
{"name": "create_handoff", "status": "success", "input": {"target": "billing_specialist"}},
],
"simulated_final_answer": "I escalated this and someone will follow up.",
},
{
"case_id": "cancel_subscription_missing_step",
"workflow": "billing",
"agent_type": "triage_agent",
"customer_request": "Please cancel my subscription at the end of this billing cycle.",
"required_step": "lookup_subscription",
"expected_handoff": "none",
"reference_answer": "I located your subscription and set it to cancel at cycle end.",
"max_retries": 1,
"simulated_retry_count": 0,
"simulated_handoff_target": "none",
"simulated_steps_completed": ["respond_to_customer"],
"simulated_tool_calls": [
{"name": "lookup_subscription", "status": "failed", "input": {"subscription_id": "sub_01"}},
],
"simulated_final_answer": "Your subscription is all set and should stop soon.",
},
{
"case_id": "invoice_request_retry_exceeded",
"workflow": "billing",
"agent_type": "billing_agent",
"customer_request": "Can you send me the invoice for March?",
"required_step": "lookup_invoice",
"expected_handoff": "none",
"reference_answer": "I found your March invoice and sent it to your email.",
"max_retries": 1,
"simulated_retry_count": 3,
"simulated_handoff_target": "none",
"simulated_steps_completed": ["lookup_invoice", "lookup_invoice", "lookup_invoice", "respond_to_customer"],
"simulated_tool_calls": [
{"name": "lookup_invoice", "status": "failed", "input": {"month": "March"}},
{"name": "lookup_invoice", "status": "failed", "input": {"month": "March"}},
{"name": "lookup_invoice", "status": "success", "input": {"month": "March"}},
],
"simulated_final_answer": "I finally found the invoice after several retries and sent it.",
},
]
# Additional comparable scenarios so RULER can rank 3-5 similar answers per group.
comparative_variants = []
for i, tone in enumerate([
"empathetic_and_precise",
"neutral_and_correct",
"vague_and_weak",
], start=1):
template = deepcopy(base_cases[0])
template["case_id"] = f"refund_double_charge_variant_{i}"
if tone == "empathetic_and_precise":
template["simulated_final_answer"] = "I understand how frustrating a double charge is. I confirmed the duplicate charge on order 8129 and escalated it to billing for immediate refund review."
elif tone == "neutral_and_correct":
template["simulated_final_answer"] = "Duplicate charge on order 8129 was confirmed and the case has been handed to billing for refund processing."
else:
template["simulated_final_answer"] = "We will look into it and get back to you."
comparative_variants.append(template)
cases = base_cases + comparative_variants
for case in cases:
case["task_definition"] = infer_task_definition(case)
case["expected_plan"] = infer_expected_plan(case)
case["expected_tools"] = infer_expected_tools(case)
case["scenario_group_id"] = f"{case['workflow']}::{infer_issue_type(case)}::{case['expected_handoff']}"
case["simulated_tool_calls"] = [enrich_tool_call(case, c) for c in case.get("simulated_tool_calls", [])]
print(f"Prepared enriched cases: {len(cases)}")
cases[0]
Prepared enriched cases: 8
{'case_id': 'refund_double_charge',
'workflow': 'billing',
'agent_type': 'triage_agent',
'customer_request': 'I was charged twice for order 8129. Please fix it.',
'required_step': 'lookup_order',
'expected_handoff': 'billing_specialist',
'reference_answer': 'I found the duplicate charge and handed this to billing for refund review.',
'max_retries': 2,
'simulated_retry_count': 1,
'simulated_handoff_target': 'billing_specialist',
'simulated_steps_completed': ['lookup_order',
'verify_duplicate_charge',
'handoff_to_billing_specialist',
'respond_to_customer'],
'simulated_tool_calls': [{'name': 'lookup_order',
'status': 'success',
'input': {'order_id': '8129'},
'output': {'status': 'success'},
'description': 'Support tool in billing workflow.'},
{'name': 'create_handoff',
'status': 'success',
'input': {'target': 'billing_specialist'},
'output': {'status': 'success'},
'description': 'Support tool in billing workflow.'}],
'simulated_final_answer': 'I confirmed the duplicate charge for order 8129 and escalated this to billing for refund processing. You will receive an update within one business day.',
'task_definition': 'Resolve this billing support request with the required steps, correct handoff (billing_specialist), and a clear final answer.',
'expected_plan': ['lookup_order',
'handoff_to_billing_specialist',
'respond_to_customer'],
'expected_tools': [{'name': 'lookup_order',
'description': 'Core billing lookup step.',
'input': {'case_id': 'refund_double_charge', 'workflow': 'billing'},
'output': {}},
{'name': 'create_handoff',
'description': 'Escalation handoff tool.',
'input': {'target': 'billing_specialist', 'reason': 'billing'},
'output': {}}],
'scenario_group_id': 'billing::billing_issue::billing_specialist'}
{'retry_budget_respected': True,
'correct_handoff': True,
'required_step_completed': True,
'required_tool_succeeded': True,
'final_answer_sufficient': {'score': 0.43177048236403187,
'reason': "The Actual Output acknowledges the issue but lacks specificity and actionable steps. It does not directly address the customer's request for a resolution, as it fails to mention the duplicate charge verification or the handoff to the billing specialist. While the handoff was correctly observed, the response is vague and does not provide immediate reassurance or clarity on the next steps, unlike the Reference answer."},
'final_answer_correct': {'score': 0.6528943405057316,
'reason': "The Actual Output does not match the Expected Output, as it lacks the confirmation of finding the duplicate charge and the handoff for refund review. However, the workflow was followed correctly with all required steps completed, including the lookup and handoff. The discrepancy in the final answer is not justified by the workflow details, as the Expected Output provides a more complete response to the customer's request."}}
ERROR:opentelemetry.sdk._shared_internal:Exception while exporting Span.
Traceback (most recent call last):
File "/usr/local/lib/python3.12/dist-packages/urllib3/connectionpool.py", line 534, in _make_request
response = conn.getresponse()
^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/dist-packages/urllib3/connection.py", line 565, in getresponse
httplib_response = super().getresponse()
^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib/python3.12/http/client.py", line 1450, in getresponse
response.begin()
File "/usr/lib/python3.12/http/client.py", line 336, in begin
version, status, reason = self._read_status()
^^^^^^^^^^^^^^^^^^^
File "/usr/lib/python3.12/http/client.py", line 297, in _read_status
line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1")
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib/python3.12/socket.py", line 720, in readinto
return self._sock.recv_into(b)
^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib/python3.12/ssl.py", line 1251, in recv_into
return self.read(nbytes, buffer)
^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib/python3.12/ssl.py", line 1103, in read
return self._sslobj.read(len, buffer)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
TimeoutError: The read operation timed out
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/usr/local/lib/python3.12/dist-packages/requests/adapters.py", line 667, in send
resp = conn.urlopen(
^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/dist-packages/urllib3/connectionpool.py", line 841, in urlopen
retries = retries.increment(
^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/dist-packages/urllib3/util/retry.py", line 474, in increment
raise reraise(type(error), error, _stacktrace)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/dist-packages/urllib3/util/util.py", line 39, in reraise
raise value
File "/usr/local/lib/python3.12/dist-packages/urllib3/connectionpool.py", line 787, in urlopen
response = self._make_request(
^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/dist-packages/urllib3/connectionpool.py", line 536, in _make_request
self._raise_timeout(err=e, url=url, timeout_value=read_timeout)
File "/usr/local/lib/python3.12/dist-packages/urllib3/connectionpool.py", line 367, in _raise_timeout
raise ReadTimeoutError(
urllib3.exceptions.ReadTimeoutError: HTTPSConnectionPool(host='cloud.langfuse.com', port=443): Read timed out. (read timeout=4.999996662139893)
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/local/lib/python3.12/dist-packages/opentelemetry/sdk/_shared_internal/__init__.py", line 179, in _export
self._exporter.export(
File "/usr/local/lib/python3.12/dist-packages/opentelemetry/exporter/otlp/proto/http/trace_exporter/__init__.py", line 182, in export
resp = self._export(serialized_data, deadline_sec - time())
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/dist-packages/opentelemetry/exporter/otlp/proto/http/trace_exporter/__init__.py", line 157, in _export
resp = self._session.post(
^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/dist-packages/requests/sessions.py", line 637, in post
return self.request("POST", url, data=data, json=json, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/dist-packages/requests/sessions.py", line 589, in request
resp = self.send(prep, **send_kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/dist-packages/requests/sessions.py", line 703, in send
r = adapter.send(request, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.12/dist-
Output()
Output()
Output()
Output()
Output()
Output()
Output()
Output()
{'trace_id': '86712dc66d68c17630301a01cf54b8f4',
'case_id': 'refund_double_charge_variant_3',
'workflow': 'billing',
'agent_type': 'triage_agent',
'scenario_group_id': 'billing::billing_issue::billing_specialist',
'payload': {'case_id': 'refund_double_charge_variant_3',
'workflow': 'billing',
'agent_type': 'triage_agent',
'customer_request': 'I was charged twice for order 8129. Please fix it.',
'required_step': 'lookup_order',
'expected_handoff': 'billing_specialist',
'reference_answer': 'I found the duplicate charge and handed this to billing for refund review.',
'max_retries': 2,
'retry_count': 1,
'handoff_target': 'billing_specialist',
'steps_completed': ['lookup_order',
'verify_duplicate_charge',
'handoff_to_billing_specialist',
'respond_to_customer'],
'tool_calls': [{'name': 'lookup_order',
'status': 'success',
'input': {'order_id': '8129'},
'output': {'status': 'success'},
'description': 'Support tool in billing workflow.'},
{'name': 'create_handoff',
'status': 'success',
'input': {'target': 'billing_specialist'},
'output': {'status': 'success'},
'description': 'Support tool in billing workflow.'}],
'final_answer': 'We will look into it and get back to you.',
'task_definition': 'Resolve this billing support request with the required steps, correct handoff (billing_specialist), and a clear final answer.',
'expected_plan': ['lookup_order',
'handoff_to_billing_specialist',
'respond_to_customer'],
'expected_tools': [{'name': 'lookup_order',
'input': {'case_id': 'refund_double_charge_variant_3',
'workflow': 'billing'},
'output': {},
'description': 'Core billing lookup step.'},
{'name': 'create_handoff',
'input': {'target': 'billing_specialist', 'reason': 'billing'},
'output': {},
'description': 'Escalation handoff tool.'}],
'scenario_group_id': 'billing::billing_issue::billing_specialist'},
'scores': {'retry_budget_respected': 1.0,
'correct_handoff': 1.0,
'required_step_completed': 1.0,
'final_answer_sufficient': 0.48608231468037266,
'final_answer_correct': 0.6493722174093651,
'task_completion': 0.6,
'argument_correctness': 0.0,
'step_efficiency': 0.6000000000000001},
'reasons': {'final_answer_sufficient': "The Actual Output acknowledges the issue but lacks specificity and actionable steps. It does not directly address the customer's request for a fix or provide immediate solutions. While the correct handoff to the billing specialist was made, the response is vague and does not confirm the duplicate charge was found or that a refund review is underway, as indicated in the reference answer.",
'final_answer_correct': "The Actual Output does not match the Expected Output exactly, as it lacks specific information about finding the duplicate charge and the handoff for refund review. However, the workflow steps were completed correctly, including the required 'lookup_order' and the correct handoff to 'billing_specialist'. The discrepancy in the final answer is not fully justified by the Input, as the Expected Output provides a more detailed response.",
'task_completion': 'The system successfully initiated the process by looking up the order and creating a handoff for further investigation. However, it did not provide a clear final answer or complete all required steps to fully resolve the billing support request.',
'argument_correctness': 'The score is 0.00 because the tool calls were incorrect due to missing input parameters, which are essential for processing the request accurately.',
'step_efficiency': 'Fallback score: StepEfficiencyMetric failed internally (UnboundLocalError). Derived from required-step presence, extra steps (4), retries (1/2).'}}
WRITE_SCORES_TO_LANGFUSE = False
if WRITE_SCORES_TO_LANGFUSE:
for evaluated in evaluated_traces:
for metric_name, metric_value in evaluated["scores"].items():
kwargs = {
"trace_id": evaluated["trace_id"],
"name": metric_name,
"value": float(metric_value),
}
if metric_name in evaluated["reasons"]:
kwargs["comment"] = evaluated["reasons"][metric_name]
langfuse.create_score(**kwargs)
print(f"Pushed scores for {len(evaluated_traces)} traces.")
else:
print("Skipping Langfuse score write-back. Set WRITE_SCORES_TO_LANGFUSE = True to enable it.")
Skipping Langfuse score write-back. Set WRITE_SCORES_TO_LANGFUSE = True to enable it.
import langsmith as ls
from langsmith import traceable
@traceable
def handle(customer_id: str, text: str) -> str:
return text
### Skip tracing entirely (PII, zero-retention tenant, etc.)
with ls.tracing_context(enabled=False):
handle("regulated-tenant", "...")
### Route to a tenant- or region-specific project with tags and metadata
with ls.tracing_context(
enabled=True,
project_name="acme-support",
tags=["production", "region-us"],
metadata={"customer_id": "acme"},
):
handle("acme", "...")
%pip install langsmith openai deepeval
# Store your secrets in a local .env file or your notebook environment.
# Do not hardcode credentials directly in the notebook.
import os
from dotenv import load_dotenv
load_dotenv()
# US-hosted LangSmith (default). For EU, set LANGCHAIN_ENDPOINT=https://eu.api.smith.langchain.com
LS_API_URL = os.getenv("LANGCHAIN_ENDPOINT", "https://api.smith.langchain.com")
os.environ["LANGCHAIN_ENDPOINT"] = LS_API_URL
api_key = os.getenv("LANGCHAIN_API_KEY") or os.getenv("LANGSMITH_API_KEY")
if not api_key:
raise ValueError(
"Set LANGCHAIN_API_KEY (or LANGSMITH_API_KEY) in your environment before running this notebook."
)
os.environ["LANGCHAIN_API_KEY"] = api_key
os.environ["LANGSMITH_API_KEY"] = api_key
# LangSmith project for this cookbook — set here (no LANGCHAIN_PROJECT env var required).
# Use "default" unless you created a dedicated project in the UI first. A non-existent name often
# causes LangSmithError 403 on GET /sessions when the SDK resolves the project.
# EU workspace: set LANGCHAIN_ENDPOINT=https://eu.api.smith.langchain.com above.
LANGSMITH_PROJECT_NAME = "default"
os.environ["LANGCHAIN_PROJECT"] = LANGSMITH_PROJECT_NAME
# Enable tracing for @traceable (LangSmith accepts LANGCHAIN_* and LANGSMITH_* mirrors)
os.environ["LANGCHAIN_TRACING_V2"] = "true"
os.environ["LANGSMITH_TRACING_V2"] = "true"
from dotenv import load_dotenv
import os
load_dotenv()
OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")
if not OPENAI_API_KEY:
print("OPENAI_API_KEY not found.")
OPENAI_API_KEY = input("Enter OPENAI_API_KEY: ").strip()
os.environ["OPENAI_API_KEY"] = OPENAI_API_KEY
print("OPENAI_API_KEY is set.")
print("LangSmith credentials are set.")
print(f"LangSmith project (this notebook): {os.environ['LANGCHAIN_PROJECT']}")
OPENAI_API_KEY is set.
LangSmith credentials are set.
LangSmith project (this notebook): default
cases = [
{
"case_id": "refund_double_charge",
"workflow": "billing",
"agent_type": "triage_agent",
"customer_request": "I was charged twice for order 8129. Please fix it.",
"required_step": "lookup_order",
"expected_handoff": "billing_specialist",
"reference_answer": "I found the duplicate charge on order 8129 and handed this to billing for refund review.",
"max_retries": 2,
"simulated_retry_count": 1,
"simulated_handoff_target": "billing_specialist",
"simulated_steps_completed": ["lookup_order", "verify_duplicate_charge", "handoff_to_billing"],
"simulated_tool_calls": [
{"name": "lookup_order", "status": "success"},
{"name": "create_billing_handoff", "status": "success"}
],
"simulated_final_answer": "I found the duplicate charge on order 8129 and handed your case to billing for refund review. You should receive an update within one business day."
},
{
"case_id": "shipping_status_no_handoff_needed",
"workflow": "shipping",
"agent_type": "triage_agent",
"customer_request": "Where is my order 9912 and when should it arrive?",
"required_step": "lookup_shipment",
"expected_handoff": "none",
"reference_answer": "Order 9912 is in transit and should arrive tomorrow based on the shipment lookup.",
"max_retries": 1,
"simulated_retry_count": 0,
"simulated_handoff_target": "none",
"simulated_steps_completed": ["lookup_shipment", "respond_to_customer"],
"simulated_tool_calls": [
{"name": "lookup_shipment", "status": "success"}
],
"simulated_final_answer": "I checked shipment 9912 and it is currently in transit. The latest carrier estimate shows delivery tomorrow."
},
{
"case_id": "reset_password_wrong_handoff",
"workflow": "account_access",
"agent_type": "triage_agent",
"customer_request": "I cannot log in after changing my password. Can someone help me reset access?",
"required_step": "lookup_account_status",
"expected_handoff": "account_security",
"reference_answer": "I checked your account status and handed the case to account security so they can restore access safely.",
"max_retries": 1,
"simulated_retry_count": 0,
"simulated_handoff_target": "billing_specialist",
"simulated_steps_completed": ["lookup_account_status", "handoff_to_billing"],
"simulated_tool_calls": [
{"name": "lookup_account_status", "status": "success"},
{"name": "create_billing_handoff", "status": "success"}
],
"simulated_final_answer": "I reviewed the account issue and escalated it for follow-up. Please wait for the next update."
},
{
"case_id": "cancel_subscription_missing_step",
"workflow": "billing",
"agent_type": "triage_agent",
"customer_request": "Please cancel my subscription at the end of this billing cycle.",
"required_step": "lookup_subscription",
"expected_handoff": "none",
"reference_answer": "I found your subscription and queued it to cancel at the end of the current billing cycle.",
"max_retries": 1,
"simulated_retry_count": 0,
"simulated_handoff_target": "none",
"simulated_steps_completed": ["respond_to_customer"],
"simulated_tool_calls": [
{"name": "lookup_subscription", "status": "failed"}
],
"simulated_final_answer": "Your subscription is all set and should stop soon."
},
{
"case_id": "invoice_request_retry_exceeded",
"workflow": "billing",
"agent_type": "billing_agent",
"customer_request": "Can you send me the invoice for March?",
"required_step": "lookup_invoice",
"expected_handoff": "none",
"reference_answer": "I found your March invoice and sent it to the email on file.",
"max_retries": 1,
"simulated_retry_count": 3,
"simulated_handoff_target": "none",
"simulated_steps_completed": ["lookup_invoice", "email_invoice"],
"simulated_tool_calls": [
{"name": "lookup_invoice", "status": "success"},
{"name": "email_invoice", "status": "success"}
],
"simulated_final_answer": "I sent your March invoice to the email on file."
},
{
"case_id": "wrong_fact_in_final_answer",
"workflow": "returns",
"agent_type": "returns_agent",
"customer_request": "What is the return deadline for order 4481?",
"required_step": "lookup_return_policy",
"expected_handoff": "none",
"reference_answer": "Order 4481 is eligible for return within 30 days of delivery.",
"max_retries": 1,
"simulated_retry_count": 0,
"simulated_handoff_target": "none",
"simulated_steps_completed": ["lookup_return_policy", "respond_to_customer"],
"simulated_tool_calls": [
{"name": "lookup_return_policy", "status": "success"}
],
"simulated_final_answer": "Order 4481 can be returned within 14 days of delivery."
}
]
for case in cases:
print(f"{case['case_id']}: workflow={case['workflow']}, agent_type={case['agent_type']}")
{
"case_id": "refund_double_charge",
"retry_count": 1,
"handoff_target": "billing_specialist",
"final_answer": "I found the duplicate charge on order 8129 and handed your case to billing for refund review. You should receive an update within one business day."
}
{
"case_id": "shipping_status_no_handoff_needed",
"retry_count": 0,
"handoff_target": "none",
"final_answer": "I checked shipment 9912 and it is currently in transit. The latest carrier estimate shows delivery tomorrow."
}
{
"case_id": "reset_password_wrong_handoff",
"retry_count": 0,
"handoff_target": "billing_specialist",
"final_answer": "I reviewed the account issue and escalated it for follow-up. Please wait for the next update."
}
{
"case_id": "cancel_subscription_missing_step",
"retry_count": 0,
"handoff_target": "none",
"final_answer": "Your subscription is all set and should stop soon."
}
{
"case_id": "invoice_request_retry_exceeded",
"retry_count": 3,
"handoff_target": "none",
"final_answer": "I sent your March invoice to the email on file."
}
{
"case_id": "wrong_fact_in_final_answer",
"retry_count": 0,
"handoff_target": "none",
"final_answer": "Order 4481 can be returned within 14 days of delivery."
}
Traces flushed to LangSmith.
import json
import os
from datetime import datetime, timedelta, timezone
from langsmith import Client
BATCH_SIZE = 10
TOTAL_TRACES = 100
EVAL_TAG = "ext_eval_pipelines"
WORKFLOW_FILTERS = {"billing", "shipping", "returns", "account_access"}
AGENT_TYPE_FILTERS = {"triage_agent", "billing_agent", "returns_agent"}
LS_API_URL = os.environ["LANGCHAIN_ENDPOINT"]
PROJECT_NAME = os.environ["LANGCHAIN_PROJECT"]
ls_client = Client(api_url=LS_API_URL)
now = datetime.now(timezone.utc)
five_am_today = datetime(now.year, now.month, now.day, 5, 0, tzinfo=timezone.utc)
five_am_yesterday = five_am_today - timedelta(days=1)
seven_days_ago = now - timedelta(days=7)
def parse_trace_payload(run):
out = run.outputs
if out is None:
raise TypeError("Run has no outputs")
if isinstance(out, dict):
if set(out.keys()) == {"output"} and isinstance(out["output"], dict):
return out["output"]
if isinstance(out.get("output"), dict) and "case_id" in out["output"]:
return out["output"]
if "case_id" in out and "workflow" in out:
return out
if isinstance(out, str):
return json.loads(out)
raise TypeError("Unsupported run output format")
def is_support_case_trace(run):
trace_name = getattr(run, "name", "") or ""
if trace_name.startswith("Support case:"):
return True
try:
payload = parse_trace_payload(run)
except Exception:
return False
required_keys = {"workflow", "agent_type", "required_step", "final_answer"}
return required_keys.issubset(payload.keys())
def matches_eval_filters(run):
try:
payload = parse_trace_payload(run)
except Exception:
return False
return (
payload.get("workflow") in WORKFLOW_FILTERS
and payload.get("agent_type") in AGENT_TYPE_FILTERS
)
def run_tags(run):
tags = getattr(run, "tags", None) or []
if tags:
return list(tags)
extra = getattr(run, "extra", None) or {}
return list(extra.get("tags") or [])
def run_start_utc(run):
st = run.start_time
if st is None:
return None
if isinstance(st, str):
st = datetime.fromisoformat(st.replace("Z", "+00:00"))
if st.tzinfo is None:
st = st.replace(tzinfo=timezone.utc)
return st
def in_time_window(run, from_ts, to_ts):
st = run_start_utc(run)
if st is None:
return True
return from_ts <= st <= to_ts
def fetch_support_traces(from_timestamp, to_timestamp, verbose=True):
"""Pull root runs; prefer server-side tag filter, fall back if it returns nothing."""
def query_runs(*, tag_filter: bool, start_time, limit: int):
kw = dict(
project_name=PROJECT_NAME,
start_time=start_time,
is_root=True,
limit=limit,
)
if tag_filter:
kw["filter"] = f'has(tags, "{EVAL_TAG}")'
return list(ls_client.list_runs(**kw))
# 1) Tag filter + API start_time (fast path when it matches your LangSmith version)
candidates = query_runs(tag_filter=True, start_time=from_timestamp, limit=TOTAL_TRACES)
if verbose and not candidates:
print(
"Note: list_runs with has(tags, ...) returned 0 rows; "
"retrying without the server tag filter and matching tags in Python."
)
if not candidates:
candidates = query_runs(tag_filter=False, start_time=from_timestamp, limit=TOTAL_TRACES)
# 2) Still empty — widen API window to 7d (busy "default" projects may need this)
if not candidates:
if verbose:
print("Note: widening API start_time to the last 7 days to find recent roots.")
candidates = query_runs(
tag_filter=False, start_time=seven_days_ago, limit=TOTAL_TRACES
)
if not candidates:
if verbose:
print(
"Note: listing the most recent root runs with no start_time filter "
f"(cap {TOTAL_TRACES}); check project name if this is still empty."
)
candidates = list(
ls_client.list_runs(
project_name=PROJECT_NAME,
is_root=True,
limit=TOTAL_TRACES,
)
)
windowed = [r for r in candidates if in_time_window(r, from_timestamp, to_timestamp)]
if not windowed and candidates:
tag_only = [r for r in candidates if EVAL_TAG in run_tags(r)]
if tag_only:
if verbose:
print(
"Note: no runs fell in the requested time window; "
"using tag-matched runs from the query batch instead."
)
windowed = tag_only
tagged = [r for r in windowed if EVAL_TAG in run_tags(r)]
raw_batch = tagged[:BATCH_SIZE] if tagged else windowed[:BATCH_SIZE]
support_traces = [t for t in raw_batch if is_support_case_trace(t)]
filtered_traces = [t for t in support_traces if matches_eval_filters(t)]
return raw_batch, support_traces, filtered_traces
raw_batch, support_traces, traces_batch = fetch_support_traces(
from_timestamp=five_am_yesterday,
to_timestamp=now,
)
if not traces_batch:
raw_batch, support_traces, traces_batch = fetch_support_traces(
from_timestamp=seven_days_ago,
to_timestamp=now,
)
print("No matching traces found in the last day, so the search was widened to the last 7 days.")
print(f"Tagged traces fetched: {len(raw_batch)}")
print(f"Support-case traces found: {len(support_traces)}")
print(f"Representative traces in first batch: {len(traces_batch)}")
if not traces_batch:
raise ValueError(
"No support-agent traces matched the evaluation filters. "
"Re-run the prep cell in this kernel, then this cell. "
f"Confirm runs land in project {PROJECT_NAME!r} (same LANGCHAIN_PROJECT as tracing) "
f"and runs include tag {EVAL_TAG!r} (see LangSmith UI)."
)
Tagged traces fetched: 10
Support-case traces found: 10
Representative traces in first batch: 10
{'retry_budget_respected': True,
'correct_handoff': True,
'required_step_completed': True,
'required_tool_succeeded': True,
'final_answer_sufficient': {'score': 0.44777806741033865,
'reason': "The Actual Output addresses the customer's request by providing a return deadline for order 4481, aligning with step 1. However, it does not match the Reference answer, indicating a discrepancy in the return policy information. The response is concise and directly answers the request, fulfilling step 4, but lacks additional workflow context or a suggested next action, which affects steps 2 and 3."},
'final_answer_correct': {'score': 0.39385741192404006,
'reason': "The Actual Output does not match the Expected Output, as it states a 14-day return period instead of the correct 30-day period. The Input aligns with the workflow details, as the required step 'lookup_return_policy' was completed successfully. However, the discrepancy in the return period is not justified by the Input or workflow details, indicating a failure in accurately retrieving or conveying the return policy information."}}
在外部给 trace 打分
使用外部评判逻辑、硬检查，或两者结合。关键是打分逻辑要放在追踪系统之外，这样它才能随你的应用一起演进。LangSmith 可以存储 run，并可选地在 run 上存储 feedback，但"什么算好的行为"由你的评估器来定义。
{'trace_id': '019d2066-0bf9-7d82-90ca-a6dc210fcb27',
'case_id': 'wrong_fact_in_final_answer',
'workflow': 'returns',
'agent_type': 'returns_agent',
'payload': {'agent_type': 'returns_agent',
'case_id': 'wrong_fact_in_final_answer',
'customer_request': 'What is the return deadline for order 4481?',
'expected_handoff': 'none',
'final_answer': 'Order 4481 can be returned within 14 days of delivery.',
'handoff_target': 'none',
'max_retries': 1,
'reference_answer': 'Order 4481 is eligible for return within 30 days of delivery.',
'required_step': 'lookup_return_policy',
'retry_count': 0,
'steps_completed': ['lookup_return_policy', 'respond_to_customer'],
'tool_calls': [{'name': 'lookup_return_policy',
'status': 'success',
'input': {},
'output': {},
'description': ''}],
'workflow': 'returns',
'task_definition': 'Resolve returns request with required step lookup_return_policy.'},
'scores': {'retry_budget_respected': 1.0,
'correct_handoff': 1.0,
'required_step_completed': 1.0,
'required_tool_succeeded': 1.0,
'final_answer_sufficient': 0.4721572213322875,
'final_answer_correct': 0.49714501380217674,
'task_completion': 0.6,
'argument_correctness': 0.0,
'step_efficiency': 1.0},
'reasons': {'final_answer_sufficient': "The Actual Output addresses the customer's request by providing a return deadline for order 4481, aligning with step 1. However, it contradicts the reference answer, which states a 30-day return period, indicating a potential error in the lookup process. The response is concise and directly answers the request, fulfilling step 4, but lacks additional context or next steps, which could enhance clarity and guidance for the customer, as per steps 2 and 3.",
'final_answer_correct': "The Actual Output does not match the Expected Output, as the return period stated is 14 days instead of the expected 30 days. The Input aligns with the workflow details, as the required step 'lookup_return_policy' was completed successfully, and no handoff was needed. However, the discrepancy in the return period is not justified by the Input or workflow details, indicating a partial misalignment in the process.",
'task_completion': 'The system correctly identified the return window for order 4481, which is a crucial part of resolving a return request. However, it did not perform the required step of looking up the return policy, which may include additional conditions or steps necessary for processing the return.',
'argument_correctness': 'The score is 0.00 because the tool call failed due to missing input parameters, which are essential for processing the request accurately.',
'step_efficiency': 'Fallback score used because StepEfficiencyMetric failed internally (UnboundLocalError). Derived from required-step presence, extra steps (2), and retry usage (0/1).'}}
import os
from dotenv import load_dotenv
load_dotenv()
# US-hosted LangSmith (default). For EU, set LANGCHAIN_ENDPOINT=https://eu.api.smith.langchain.com
LS_API_URL = os.getenv("LANGCHAIN_ENDPOINT", "https://api.smith.langchain.com")
os.environ["LANGCHAIN_ENDPOINT"] = LS_API_URL
api_key = os.getenv("LANGCHAIN_API_KEY") or os.getenv("LANGSMITH_API_KEY")
if not api_key:
raise ValueError(
"Set LANGCHAIN_API_KEY (or LANGSMITH_API_KEY) in your environment before running this notebook."
)
os.environ["LANGCHAIN_API_KEY"] = api_key
os.environ["LANGSMITH_API_KEY"] = api_key
# LangSmith project for this cookbook — set here (same as LangSmith external-eval notebook).
# Use "default" unless you created a dedicated project in the UI first.
LANGSMITH_PROJECT_NAME = "default"
os.environ["LANGCHAIN_PROJECT"] = LANGSMITH_PROJECT_NAME
# Enable tracing for @traceable (mirrors LangSmith external-eval notebook)
os.environ["LANGCHAIN_TRACING_V2"] = "true"
os.environ["LANGSMITH_TRACING_V2"] = "true"
from dotenv import load_dotenv
import os
load_dotenv()
OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")
if not OPENAI_API_KEY:
print("OPENAI_API_KEY not found.")
OPENAI_API_KEY = input("Enter OPENAI_API_KEY: ").strip()
os.environ["OPENAI_API_KEY"] = OPENAI_API_KEY
print("OPENAI_API_KEY is set.")
print("LangSmith credentials are set.")
print(f"LangSmith project (this notebook): {os.environ['LANGCHAIN_PROJECT']}")
OPENAI_API_KEY is set.
LangSmith credentials are set.
LangSmith project (this notebook): default
import json
import os
from datetime import datetime, timedelta, timezone
from collections import defaultdict
from langsmith import Client
BATCH_SIZE = 10
TOTAL_TRACES = 100
EVAL_TAG = "ext_eval_pipelines"
WORKFLOW_FILTERS = {"billing", "shipping", "returns", "account_access"}
AGENT_TYPE_FILTERS = {"triage_agent", "billing_agent", "returns_agent"}
LS_API_URL = os.environ["LANGCHAIN_ENDPOINT"]
PROJECT_NAME = os.environ["LANGCHAIN_PROJECT"]
ls_client = Client(api_url=LS_API_URL)
now = datetime.now(timezone.utc)
five_am_today = datetime(now.year, now.month, now.day, 5, 0, tzinfo=timezone.utc)
five_am_yesterday = five_am_today - timedelta(days=1)
seven_days_ago = now - timedelta(days=7)
def parse_trace_payload(run):
out = run.outputs
if out is None:
raise TypeError("Run has no outputs")
if isinstance(out, dict):
if set(out.keys()) == {"output"} and isinstance(out["output"], dict):
return out["output"]
if isinstance(out.get("output"), dict) and "case_id" in out["output"]:
return out["output"]
if "case_id" in out and "workflow" in out:
return out
if isinstance(out, str):
return json.loads(out)
raise TypeError("Unsupported run output format")
def is_support_case_trace(run):
trace_name = getattr(run, "name", "") or ""
if trace_name.startswith("Support case:"):
return True
try:
payload = parse_trace_payload(run)
except Exception:
return False
required_keys = {"workflow", "agent_type", "required_step", "final_answer"}
return required_keys.issubset(payload.keys())
def matches_eval_filters(run):
try:
payload = parse_trace_payload(run)
except Exception:
return False
return (
payload.get("workflow") in WORKFLOW_FILTERS
and payload.get("agent_type") in AGENT_TYPE_FILTERS
)
def run_tags(run):
tags = getattr(run, "tags", None) or []
if tags:
return list(tags)
extra = getattr(run, "extra", None) or {}
return list(extra.get("tags") or [])
def run_start_utc(run):
st = run.start_time
if st is None:
return None
if isinstance(st, str):
st = datetime.fromisoformat(st.replace("Z", "+00:00"))
if st.tzinfo is None:
st = st.replace(tzinfo=timezone.utc)
return st
def in_time_window(run, from_ts, to_ts):
st = run_start_utc(run)
if st is None:
return True
return from_ts <= st <= to_ts
def fetch_support_traces(from_timestamp, to_timestamp, verbose=True):
def query_runs(*, tag_filter: bool, start_time, limit: int):
kw = dict(
project_name=PROJECT_NAME,
start_time=start_time,
is_root=True,
limit=limit,
)
if tag_filter:
kw["filter"] = f'has(tags, "{EVAL_TAG}")'
return list(ls_client.list_runs(**kw))
candidates = query_runs(tag_filter=True, start_time=from_timestamp, limit=TOTAL_TRACES)
if verbose and not candidates:
print(
"Note: list_runs with has(tags, ...) returned 0 rows; "
"retrying without the server tag filter and matching tags in Python."
)
if not candidates:
candidates = query_runs(tag_filter=False, start_time=from_timestamp, limit=TOTAL_TRACES)
if not candidates:
if verbose:
print("Note: widening API start_time to the last 7 days to find recent roots.")
candidates = query_runs(tag_filter=False, start_time=seven_days_ago, limit=TOTAL_TRACES)
if not candidates:
if verbose:
print(
"Note: listing the most recent root runs with no start_time filter "
f"(cap {TOTAL_TRACES}); check project name if this is still empty."
)
candidates = list(
ls_client.list_runs(project_name=PROJECT_NAME, is_root=True, limit=TOTAL_TRACES)
)
windowed = [r for r in candidates if in_time_window(r, from_timestamp, to_timestamp)]
if not windowed and candidates:
tag_only = [r for r in candidates if EVAL_TAG in run_tags(r)]
if tag_only:
if verbose:
print(
"Note: no runs fell in the requested time window; "
"using tag-matched runs from the query batch instead."
)
windowed = tag_only
tagged = [r for r in windowed if EVAL_TAG in run_tags(r)]
raw_batch = tagged[:BATCH_SIZE] if tagged else windowed[:BATCH_SIZE]
support_traces = [t for t in raw_batch if is_support_case_trace(t)]
filtered_traces = [t for t in support_traces if matches_eval_filters(t)]
return raw_batch, support_traces, filtered_traces
raw_batch, support_traces, traces_batch = fetch_support_traces(
from_timestamp=five_am_yesterday,
to_timestamp=now,
)
if not traces_batch:
raw_batch, support_traces, traces_batch = fetch_support_traces(
from_timestamp=seven_days_ago,
to_timestamp=now,
)
print("No matching traces found in the last day, so the search was widened to the last 7 days.")
print(f"Tagged traces fetched: {len(raw_batch)}")
print(f"Support-case traces found: {len(support_traces)}")
print(f"Representative traces in first batch: {len(traces_batch)}")
if not traces_batch:
raise ValueError(
"No support-agent traces matched the evaluation filters. "
"Re-run the logging cells in ch09_example_external_evaluation_pipelines_langsmith.ipynb, then this cell. "
f"Confirm runs land in project {PROJECT_NAME!r} (same LANGCHAIN_PROJECT as tracing) "
f"and runs include tag {EVAL_TAG!r}."
)
traces = [
{"trace_id": str(run.id), "payload": parse_trace_payload(run)}
for run in traces_batch
]
print(f"Rows for RULER grouping: {len(traces)}")
Tagged traces fetched: 10
Support-case traces found: 10
Representative traces in first batch: 10
Rows for RULER grouping: 10
def scenario_group_id(payload):
return payload.get("scenario_group_id") or (
f"{payload.get('workflow','unknown')}::{payload.get('required_step','unknown')}::{payload.get('expected_handoff','none')}"
)
groups = defaultdict(list)
for row in traces:
groups[scenario_group_id(row["payload"])].append(row)
grouped = {gid: rows[:5] for gid, rows in groups.items() if len(rows) >= 3}
print(f"Comparable groups (>=3 traces): {len(grouped)}")
Comparable groups (>=3 traces): 0
import asyncio
import nest_asyncio
from art.rewards import ruler
def build_messages(payload):
return [
{"role": "system", "content": "Judge answer text quality for customer support."},
{
"role": "user",
"content": (
f"Customer request: {payload.get('customer_request','')}\n"
f"Workflow: {payload.get('workflow','')}\n"
f"Required step: {payload.get('required_step','')}"
),
},
{"role": "assistant", "content": payload.get("final_answer", "")},
]
async def run_ruler(grouped_rows, judge_model="openai/o3"):
ranked_rows = []
for gid, rows in grouped_rows.items():
message_lists = [build_messages(r["payload"]) for r in rows]
scores = await ruler(
message_lists,
judge_model,
rubric=(
"Rank by empathy, technical relevance, policy correctness, and actionability. "
"Prefer clear next steps. Penalize vague answers."
),
)
tmp = []
for i, score in enumerate(scores):
tmp.append(
{
"group_id": gid,
"trace_id": rows[i]["trace_id"],
"case_id": rows[i]["payload"].get("case_id"),
"final_answer": rows[i]["payload"].get("final_answer"),
"ruler_score": score.score,
"ruler_explanation": score.explanation,
}
)
tmp.sort(key=lambda x: x["ruler_score"], reverse=True)
for rank, row in enumerate(tmp, start=1):
row["rank_in_group"] = rank
row["weak_answer_flag"] = rank == len(tmp)
ranked_rows.extend(tmp)
return ranked_rows
nest_asyncio.apply()
ruler_rankings = asyncio.run(run_ruler(grouped))
print(f"Total ranked rows: {len(ruler_rankings)}")
Total ranked rows: 0
weak_answers = [r for r in ruler_rankings if r["weak_answer_flag"]]
weak_answers[:10]
