mirror of
https://github.com/hanzoai/playwright.git
synced 2026-09-24 17:01:53 +00:00
test(mcp): introduce eval tests (#38318)
This commit is contained in:
Dmitry Gozman
+1
-1
@@ -30,7 +30,7 @@
|
||||
"test": "playwright test --config=tests/library/playwright.config.ts",
|
||||
"test-mcp": "playwright test --config=tests/mcp/playwright.config.ts",
|
||||
"ctest-mcp": "playwright test --config=tests/mcp/playwright.config.ts --project=chrome",
|
||||
"etest-mcp": "playwright test --config=tests/mcp/playwright.config.extension.ts",
|
||||
"eval-mcp": "playwright test --config=tests/mcp/playwright.config.ts --project=eval",
|
||||
"eslint": "eslint --cache",
|
||||
"tsc": "tsc -p . && tsc -p packages/html-reporter/",
|
||||
"doc": "node utils/doclint/cli.js",
|
||||
|
||||
@@ -52,8 +52,9 @@ export class Agent<T extends z.ZodSchema<any>> {
|
||||
try {
|
||||
return await this.loop.run<z.output<T>>(`${prompt}\n\nTask:\n${task}\n\nParams:\n${JSON.stringify(params, null, 2)}`, {
|
||||
...options,
|
||||
tools,
|
||||
callTool,
|
||||
// TODO: fix types in tiny-loop
|
||||
tools: tools as any,
|
||||
callTool: callTool as any,
|
||||
resultSchema: this.resultSchema
|
||||
});
|
||||
} finally {
|
||||
|
||||
@@ -36,8 +36,9 @@ export async function performTask(context: playwright.BrowserContext, task: stri
|
||||
|
||||
try {
|
||||
return await loop.run(task, {
|
||||
tools: await backend.listTools(),
|
||||
callTool,
|
||||
// TODO: fix types in tiny-loop
|
||||
tools: await backend.listTools() as any,
|
||||
callTool: callTool as any,
|
||||
logger,
|
||||
});
|
||||
} finally {
|
||||
|
||||
@@ -0,0 +1,369 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>Order Confirmation</title>
|
||||
<style>
|
||||
* {
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
box-sizing: border-box;
|
||||
}
|
||||
|
||||
body {
|
||||
font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif;
|
||||
background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
|
||||
padding: 20px;
|
||||
min-height: 100vh;
|
||||
display: flex;
|
||||
justify-content: center;
|
||||
align-items: center;
|
||||
}
|
||||
|
||||
.container {
|
||||
background: white;
|
||||
border-radius: 10px;
|
||||
box-shadow: 0 10px 40px rgba(0, 0, 0, 0.2);
|
||||
padding: 40px;
|
||||
max-width: 700px;
|
||||
width: 100%;
|
||||
}
|
||||
|
||||
.success-icon {
|
||||
text-align: center;
|
||||
margin-bottom: 20px;
|
||||
}
|
||||
|
||||
.success-icon svg {
|
||||
width: 80px;
|
||||
height: 80px;
|
||||
stroke: #4CAF50;
|
||||
stroke-width: 2;
|
||||
stroke-linecap: round;
|
||||
stroke-linejoin: round;
|
||||
fill: none;
|
||||
}
|
||||
|
||||
h1 {
|
||||
color: #333;
|
||||
margin-bottom: 10px;
|
||||
text-align: center;
|
||||
font-size: 28px;
|
||||
}
|
||||
|
||||
.subtitle {
|
||||
text-align: center;
|
||||
color: #666;
|
||||
margin-bottom: 30px;
|
||||
font-size: 16px;
|
||||
}
|
||||
|
||||
.order-details {
|
||||
background: #f8f9fa;
|
||||
border-radius: 8px;
|
||||
padding: 25px;
|
||||
margin-bottom: 25px;
|
||||
}
|
||||
|
||||
.detail-section {
|
||||
margin-bottom: 20px;
|
||||
}
|
||||
|
||||
.detail-section:last-child {
|
||||
margin-bottom: 0;
|
||||
}
|
||||
|
||||
.section-title {
|
||||
font-size: 14px;
|
||||
font-weight: 600;
|
||||
color: #888;
|
||||
text-transform: uppercase;
|
||||
margin-bottom: 10px;
|
||||
letter-spacing: 0.5px;
|
||||
}
|
||||
|
||||
.detail-row {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
padding: 8px 0;
|
||||
border-bottom: 1px solid #e0e0e0;
|
||||
}
|
||||
|
||||
.detail-row:last-child {
|
||||
border-bottom: none;
|
||||
}
|
||||
|
||||
.detail-label {
|
||||
color: #555;
|
||||
font-weight: 500;
|
||||
}
|
||||
|
||||
.detail-value {
|
||||
color: #333;
|
||||
font-weight: 400;
|
||||
text-align: right;
|
||||
}
|
||||
|
||||
.price-section {
|
||||
background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
|
||||
color: white;
|
||||
border-radius: 8px;
|
||||
padding: 20px;
|
||||
margin-bottom: 25px;
|
||||
}
|
||||
|
||||
.price-row {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
padding: 8px 0;
|
||||
}
|
||||
|
||||
.total-price {
|
||||
font-size: 32px;
|
||||
font-weight: 700;
|
||||
margin-top: 10px;
|
||||
}
|
||||
|
||||
.delivery-section {
|
||||
background: #e3f2fd;
|
||||
border-left: 4px solid #2196F3;
|
||||
padding: 20px;
|
||||
border-radius: 5px;
|
||||
margin-bottom: 25px;
|
||||
}
|
||||
|
||||
.delivery-section h3 {
|
||||
color: #1976D2;
|
||||
margin-bottom: 10px;
|
||||
font-size: 18px;
|
||||
}
|
||||
|
||||
.delivery-info {
|
||||
color: #555;
|
||||
line-height: 1.6;
|
||||
}
|
||||
|
||||
.delivery-days {
|
||||
font-size: 24px;
|
||||
font-weight: 700;
|
||||
color: #1976D2;
|
||||
}
|
||||
|
||||
.btn-back {
|
||||
width: 100%;
|
||||
padding: 15px;
|
||||
background: #e0e0e0;
|
||||
color: #333;
|
||||
border: none;
|
||||
border-radius: 5px;
|
||||
font-size: 16px;
|
||||
font-weight: 600;
|
||||
cursor: pointer;
|
||||
transition: all 0.2s;
|
||||
}
|
||||
|
||||
.btn-back:hover {
|
||||
background: #d0d0d0;
|
||||
}
|
||||
|
||||
.error-message {
|
||||
background: #ffebee;
|
||||
color: #c62828;
|
||||
padding: 20px;
|
||||
border-radius: 8px;
|
||||
border-left: 4px solid #c62828;
|
||||
text-align: center;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="container" id="content">
|
||||
<!-- Content will be generated by JavaScript -->
|
||||
</div>
|
||||
|
||||
<script>
|
||||
// Product prices in USD
|
||||
const productPrices = {
|
||||
'table': 299.99,
|
||||
'chair': 89.99,
|
||||
'sofa': 699.99,
|
||||
'desk': 249.99,
|
||||
'cabinet': 399.99
|
||||
};
|
||||
|
||||
// Calculate delivery days based on ZIP code
|
||||
function calculateDeliveryDays(zipCode) {
|
||||
if (!zipCode) return 7;
|
||||
|
||||
// Simple logic: use first digit of ZIP to determine delivery zone
|
||||
const firstDigit = parseInt(zipCode.toString().charAt(0));
|
||||
|
||||
// ZIP codes 0-2: East Coast
|
||||
// ZIP codes 3-5: Central
|
||||
// ZIP codes 6-9: West Coast
|
||||
|
||||
if (firstDigit >= 0 && firstDigit <= 2) {
|
||||
return 3;
|
||||
} else if (firstDigit >= 3 && firstDigit <= 5) {
|
||||
return 5;
|
||||
} else if (firstDigit >= 6 && firstDigit <= 9) {
|
||||
return 7;
|
||||
}
|
||||
|
||||
return 7; // Default
|
||||
}
|
||||
|
||||
// Get URL parameters
|
||||
function getUrlParams() {
|
||||
// Handle both URL search params and hash params
|
||||
let queryString = window.location.search;
|
||||
if (!queryString && window.location.hash) {
|
||||
// Extract query string from hash (format: #?param=value)
|
||||
queryString = window.location.hash.substring(2); // Remove '#?'
|
||||
}
|
||||
const params = new URLSearchParams(queryString);
|
||||
return {
|
||||
productType: params.get('productType'),
|
||||
color: params.get('color'),
|
||||
name: params.get('name'),
|
||||
email: params.get('email'),
|
||||
address: params.get('address'),
|
||||
city: params.get('city'),
|
||||
state: params.get('state'),
|
||||
zipCode: params.get('zipCode')
|
||||
};
|
||||
}
|
||||
|
||||
// Format currency
|
||||
function formatCurrency(amount) {
|
||||
return new Intl.NumberFormat('en-US', {
|
||||
style: 'currency',
|
||||
currency: 'USD'
|
||||
}).format(amount);
|
||||
}
|
||||
|
||||
// Capitalize first letter
|
||||
function capitalize(str) {
|
||||
if (!str) return '';
|
||||
return str.charAt(0).toUpperCase() + str.slice(1);
|
||||
}
|
||||
|
||||
// Generate confirmation page
|
||||
function generateConfirmation() {
|
||||
const params = getUrlParams();
|
||||
const container = document.getElementById('content');
|
||||
|
||||
// Check if we have required parameters
|
||||
if (!params.productType || !params.name) {
|
||||
container.innerHTML = `
|
||||
<div class="error-message">
|
||||
<h2>Order Information Missing</h2>
|
||||
<p>Unable to process order confirmation. Please return to the order form.</p>
|
||||
<button class="btn-back" onclick="window.history.back()">Go Back</button>
|
||||
</div>
|
||||
`;
|
||||
return;
|
||||
}
|
||||
|
||||
// Calculate price and delivery
|
||||
const basePrice = productPrices[params.productType] || 0;
|
||||
const tax = basePrice * 0.08; // 8% tax
|
||||
const shipping = 29.99;
|
||||
const totalPrice = basePrice + tax + shipping;
|
||||
const deliveryDays = calculateDeliveryDays(params.zipCode);
|
||||
|
||||
// Generate HTML
|
||||
container.innerHTML = `
|
||||
<div class="success-icon">
|
||||
<svg viewBox="0 0 52 52">
|
||||
<circle cx="26" cy="26" r="25"/>
|
||||
<path d="M14.1 27.2l7.1 7.2 16.7-16.8"/>
|
||||
</svg>
|
||||
</div>
|
||||
|
||||
<h1>Order Confirmed!</h1>
|
||||
<p class="subtitle">Thank you for your purchase, ${params.name}</p>
|
||||
|
||||
<div class="order-details">
|
||||
<div class="detail-section">
|
||||
<div class="section-title">Product Details</div>
|
||||
<div class="detail-row">
|
||||
<span class="detail-label">Product:</span>
|
||||
<span class="detail-value">${capitalize(params.productType)}</span>
|
||||
</div>
|
||||
<div class="detail-row">
|
||||
<span class="detail-label">Color:</span>
|
||||
<span class="detail-value">${capitalize(params.color)}</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="detail-section">
|
||||
<div class="section-title">Customer Information</div>
|
||||
<div class="detail-row">
|
||||
<span class="detail-label">Name:</span>
|
||||
<span class="detail-value">${params.name}</span>
|
||||
</div>
|
||||
<div class="detail-row">
|
||||
<span class="detail-label">Email:</span>
|
||||
<span class="detail-value">${params.email}</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="detail-section">
|
||||
<div class="section-title">Shipping Address</div>
|
||||
<div class="detail-row">
|
||||
<span class="detail-label">Address:</span>
|
||||
<span class="detail-value">${params.address}</span>
|
||||
</div>
|
||||
<div class="detail-row">
|
||||
<span class="detail-label">City:</span>
|
||||
<span class="detail-value">${params.city}</span>
|
||||
</div>
|
||||
<div class="detail-row">
|
||||
<span class="detail-label">State:</span>
|
||||
<span class="detail-value">${params.state}</span>
|
||||
</div>
|
||||
<div class="detail-row">
|
||||
<span class="detail-label">ZIP Code:</span>
|
||||
<span class="detail-value">${params.zipCode}</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="price-section">
|
||||
<div class="price-row">
|
||||
<span>Product Price:</span>
|
||||
<span>${formatCurrency(basePrice)}</span>
|
||||
</div>
|
||||
<div class="price-row">
|
||||
<span>Tax (8%):</span>
|
||||
<span>${formatCurrency(tax)}</span>
|
||||
</div>
|
||||
<div class="price-row">
|
||||
<span>Shipping:</span>
|
||||
<span>${formatCurrency(shipping)}</span>
|
||||
</div>
|
||||
<div class="price-row total-price">
|
||||
<span>Total:</span>
|
||||
<span>${formatCurrency(totalPrice)}</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="delivery-section">
|
||||
<h3>📦 Estimated Delivery</h3>
|
||||
<div class="delivery-info">
|
||||
<p>Your order will arrive in approximately <span class="delivery-days">${deliveryDays} days</span></p>
|
||||
<p style="margin-top: 10px; font-size: 14px;">We'll send tracking information to ${params.email} once your order ships.</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<button class="btn-back" onclick="window.location.href='./fill-form.html'">Place Another Order</button>
|
||||
`;
|
||||
}
|
||||
|
||||
// Initialize page
|
||||
document.addEventListener('DOMContentLoaded', generateConfirmation);
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,186 @@
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>Order Form</title>
|
||||
<style>
|
||||
* {
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
box-sizing: border-box;
|
||||
}
|
||||
|
||||
body {
|
||||
font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif;
|
||||
background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
|
||||
padding: 20px;
|
||||
min-height: 100vh;
|
||||
display: flex;
|
||||
justify-content: center;
|
||||
align-items: center;
|
||||
}
|
||||
|
||||
.container {
|
||||
background: white;
|
||||
border-radius: 10px;
|
||||
box-shadow: 0 10px 40px rgba(0, 0, 0, 0.2);
|
||||
padding: 40px;
|
||||
max-width: 600px;
|
||||
width: 100%;
|
||||
}
|
||||
|
||||
h1 {
|
||||
color: #333;
|
||||
margin-bottom: 30px;
|
||||
text-align: center;
|
||||
font-size: 28px;
|
||||
}
|
||||
|
||||
.form-group {
|
||||
margin-bottom: 20px;
|
||||
}
|
||||
|
||||
label {
|
||||
display: block;
|
||||
margin-bottom: 8px;
|
||||
color: #555;
|
||||
font-weight: 600;
|
||||
font-size: 14px;
|
||||
}
|
||||
|
||||
input[type="text"],
|
||||
input[type="email"],
|
||||
select,
|
||||
textarea {
|
||||
width: 100%;
|
||||
padding: 12px;
|
||||
border: 2px solid #e0e0e0;
|
||||
border-radius: 5px;
|
||||
font-size: 14px;
|
||||
transition: border-color 0.3s;
|
||||
font-family: inherit;
|
||||
}
|
||||
|
||||
input[type="text"]:focus,
|
||||
input[type="email"]:focus,
|
||||
select:focus,
|
||||
textarea:focus {
|
||||
outline: none;
|
||||
border-color: #667eea;
|
||||
}
|
||||
|
||||
textarea {
|
||||
resize: vertical;
|
||||
min-height: 80px;
|
||||
}
|
||||
|
||||
.btn-order {
|
||||
width: 100%;
|
||||
padding: 15px;
|
||||
background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
|
||||
color: white;
|
||||
border: none;
|
||||
border-radius: 5px;
|
||||
font-size: 16px;
|
||||
font-weight: 600;
|
||||
cursor: pointer;
|
||||
transition: transform 0.2s, box-shadow 0.2s;
|
||||
margin-top: 10px;
|
||||
}
|
||||
|
||||
.btn-order:hover {
|
||||
transform: translateY(-2px);
|
||||
box-shadow: 0 5px 20px rgba(102, 126, 234, 0.4);
|
||||
}
|
||||
|
||||
.btn-order:active {
|
||||
transform: translateY(0);
|
||||
}
|
||||
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="container">
|
||||
<h1>Product Order Form</h1>
|
||||
<form id="orderForm">
|
||||
|
||||
<div class="form-group">
|
||||
<label for="productType">Product Type *</label>
|
||||
<select id="productType" name="productType" required>
|
||||
<option value="">Select a product</option>
|
||||
<option value="chair">Chair</option>
|
||||
<option value="sofa">Sofa</option>
|
||||
<option value="desk">Desk</option>
|
||||
<option value="table">Table</option>
|
||||
<option value="cabinet">Cabinet</option>
|
||||
</select>
|
||||
</div>
|
||||
|
||||
<div class="form-group">
|
||||
<label for="color">Color *</label>
|
||||
<select id="color" name="color" required>
|
||||
<option value="">Select a color</option>
|
||||
<option value="red">Red</option>
|
||||
<option value="green">Green</option>
|
||||
<option value="blue">Blue</option>
|
||||
<option value="black">Black</option>
|
||||
<option value="white">White</option>
|
||||
<option value="brown">Brown</option>
|
||||
</select>
|
||||
</div>
|
||||
|
||||
<div class="form-group">
|
||||
<label for="name">Full Name *</label>
|
||||
<input type="text" id="name" name="name" placeholder="Enter your full name" required>
|
||||
</div>
|
||||
|
||||
<div class="form-group">
|
||||
<label for="email">Email Address *</label>
|
||||
<input type="email" id="email" name="email" placeholder="[email protected]" required>
|
||||
</div>
|
||||
|
||||
<div class="form-group">
|
||||
<label for="address">Street Address *</label>
|
||||
<input type="text" id="address" name="address" placeholder="123 Main Street" required>
|
||||
</div>
|
||||
|
||||
<div class="form-group">
|
||||
<label for="city">City *</label>
|
||||
<input type="text" id="city" name="city" placeholder="City" required>
|
||||
</div>
|
||||
|
||||
<div class="form-group">
|
||||
<label for="state">State/Province *</label>
|
||||
<input type="text" id="state" name="state" placeholder="State" required>
|
||||
</div>
|
||||
|
||||
<div class="form-group">
|
||||
<label for="zipCode">ZIP/Postal Code *</label>
|
||||
<input type="text" id="zipCode" name="zipCode" placeholder="12345" required>
|
||||
</div>
|
||||
|
||||
<button type="button" class="btn-order">Order</button>
|
||||
</form>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
const form = document.getElementById('orderForm');
|
||||
|
||||
const submit = () => {
|
||||
location.href = './fill-form-submit.html#?' + new URLSearchParams(new FormData(form)).toString();
|
||||
};
|
||||
|
||||
document.querySelector('.btn-order').addEventListener('click', () => {
|
||||
if (form.checkValidity()) {
|
||||
if (window.confirm(`Please confirm your order for a ${form.color.value} ${form.productType.value}.`))
|
||||
submit();
|
||||
} else {
|
||||
form.reportValidity();
|
||||
}
|
||||
});
|
||||
|
||||
form.addEventListener('submit', e => {
|
||||
e.preventDefault();
|
||||
submit();
|
||||
});
|
||||
</script>
|
||||
</body>
|
||||
@@ -0,0 +1,330 @@
|
||||
{
|
||||
"facb950333d3cbebad4cb469c846c6a03a72dce8": {
|
||||
"result": {
|
||||
"role": "assistant",
|
||||
"content": "I'll help you navigate to the form, fill it out with the specified details, and report back the total price and delivery information.",
|
||||
"toolCalls": []
|
||||
},
|
||||
"usage": {
|
||||
"input": 4484,
|
||||
"output": 132
|
||||
}
|
||||
},
|
||||
"a8e0596eb3a8ab52837b1544d1c328f0886e26a1": {
|
||||
"result": {
|
||||
"role": "assistant",
|
||||
"content": "Navigate to the form page",
|
||||
"toolCalls": [
|
||||
{
|
||||
"name": "browser_navigate",
|
||||
"arguments": {
|
||||
"url": "http://localhost:<PORT>/evals/fill-form.html"
|
||||
},
|
||||
"id": "tooluse_GPuE3K9kQwSXLSPV2UAyEw"
|
||||
}
|
||||
]
|
||||
},
|
||||
"usage": {
|
||||
"input": 4535,
|
||||
"output": 88
|
||||
}
|
||||
},
|
||||
"bbdb8dff944cf47be23aa17fa55a678a29f5cd4a": {
|
||||
"result": {
|
||||
"role": "assistant",
|
||||
"content": "Capture snapshot to see the form",
|
||||
"toolCalls": []
|
||||
},
|
||||
"usage": {
|
||||
"input": 5232,
|
||||
"output": 69
|
||||
}
|
||||
},
|
||||
"3b8c1b6c21376d9a91c21a9e33a35439a3990c54": {
|
||||
"result": {
|
||||
"role": "assistant",
|
||||
"content": "I'll fill out the form for the first item (blue table)",
|
||||
"toolCalls": []
|
||||
},
|
||||
"usage": {
|
||||
"input": 5263,
|
||||
"output": 130
|
||||
}
|
||||
},
|
||||
"40b838f47364c51fe3512a4fc385fa16763d53a4": {
|
||||
"result": {
|
||||
"role": "assistant",
|
||||
"content": "Fill form for first item - blue table",
|
||||
"toolCalls": [
|
||||
{
|
||||
"name": "browser_fill_form",
|
||||
"arguments": {
|
||||
"fields": [
|
||||
{
|
||||
"name": "Product Type",
|
||||
"ref": "e7",
|
||||
"type": "combobox",
|
||||
"value": "Table"
|
||||
},
|
||||
{
|
||||
"name": "Color",
|
||||
"ref": "e10",
|
||||
"type": "combobox",
|
||||
"value": "Blue"
|
||||
},
|
||||
{
|
||||
"name": "Full Name",
|
||||
"ref": "e13",
|
||||
"type": "textbox",
|
||||
"value": "John Doe"
|
||||
},
|
||||
{
|
||||
"name": "Email Address",
|
||||
"ref": "e16",
|
||||
"type": "textbox",
|
||||
"value": "[email protected]"
|
||||
},
|
||||
{
|
||||
"name": "Street Address",
|
||||
"ref": "e19",
|
||||
"type": "textbox",
|
||||
"value": "123 Main St"
|
||||
},
|
||||
{
|
||||
"name": "City",
|
||||
"ref": "e22",
|
||||
"type": "textbox",
|
||||
"value": "Anytown"
|
||||
},
|
||||
{
|
||||
"name": "State/Province",
|
||||
"ref": "e25",
|
||||
"type": "textbox",
|
||||
"value": "XYZ"
|
||||
},
|
||||
{
|
||||
"name": "ZIP/Postal Code",
|
||||
"ref": "e28",
|
||||
"type": "textbox",
|
||||
"value": "12345"
|
||||
}
|
||||
]
|
||||
},
|
||||
"id": "tooluse_9iiHR97qRZWzf8YVVbkHmA"
|
||||
}
|
||||
]
|
||||
},
|
||||
"usage": {
|
||||
"input": 5301,
|
||||
"output": 331
|
||||
}
|
||||
},
|
||||
"30b7c82f8a226b17b040f846903656852e7a6e36": {
|
||||
"result": {
|
||||
"role": "assistant",
|
||||
"content": "Click the Order button for the first item",
|
||||
"toolCalls": []
|
||||
},
|
||||
"usage": {
|
||||
"input": 5830,
|
||||
"output": 81
|
||||
}
|
||||
},
|
||||
"d4c841c1aa8400cc2aeaf0a4e4db2c4df625a320": {
|
||||
"result": {
|
||||
"role": "assistant",
|
||||
"content": "Click the Order button",
|
||||
"toolCalls": []
|
||||
},
|
||||
"usage": {
|
||||
"input": 5862,
|
||||
"output": 77
|
||||
}
|
||||
},
|
||||
"51ee9f20ae47d1660b4ed9dbdaf6d44b4b951339": {
|
||||
"result": {
|
||||
"role": "assistant",
|
||||
"content": "",
|
||||
"toolCalls": [
|
||||
{
|
||||
"name": "browser_click",
|
||||
"arguments": {
|
||||
"element": "Order button",
|
||||
"ref": "e29"
|
||||
},
|
||||
"id": "tooluse_WtpFqw9JScur-lAHZEsKRQ"
|
||||
}
|
||||
]
|
||||
},
|
||||
"usage": {
|
||||
"input": 5890,
|
||||
"output": 72
|
||||
}
|
||||
},
|
||||
"4493b8b9ef4b1cfaff25033b937f9a3bc81de116": {
|
||||
"result": {
|
||||
"role": "assistant",
|
||||
"content": "",
|
||||
"toolCalls": [
|
||||
{
|
||||
"name": "browser_handle_dialog",
|
||||
"arguments": {
|
||||
"accept": true
|
||||
},
|
||||
"id": "tooluse_oUddkbggQ3aEEYKh8BU8wg"
|
||||
}
|
||||
]
|
||||
},
|
||||
"usage": {
|
||||
"input": 6042,
|
||||
"output": 55
|
||||
}
|
||||
},
|
||||
"890e35cf92b660a3fa3ade73f150a929caf92a1d": {
|
||||
"result": {
|
||||
"role": "assistant",
|
||||
"content": "",
|
||||
"toolCalls": [
|
||||
{
|
||||
"name": "browser_click",
|
||||
"arguments": {
|
||||
"element": "Place Another Order button",
|
||||
"ref": "e58"
|
||||
},
|
||||
"id": "tooluse_AlBQSRtuQT2Yot3LPY5F-Q"
|
||||
}
|
||||
]
|
||||
},
|
||||
"usage": {
|
||||
"input": 6903,
|
||||
"output": 74
|
||||
}
|
||||
},
|
||||
"9bee0bcd86f45f9622412b4c01bc0fa871054aa3": {
|
||||
"result": {
|
||||
"role": "assistant",
|
||||
"content": "",
|
||||
"toolCalls": [
|
||||
{
|
||||
"name": "browser_fill_form",
|
||||
"arguments": {
|
||||
"fields": [
|
||||
{
|
||||
"name": "Product Type",
|
||||
"ref": "e7",
|
||||
"type": "combobox",
|
||||
"value": "Desk"
|
||||
},
|
||||
{
|
||||
"name": "Color",
|
||||
"ref": "e10",
|
||||
"type": "combobox",
|
||||
"value": "Green"
|
||||
},
|
||||
{
|
||||
"name": "Full Name",
|
||||
"ref": "e13",
|
||||
"type": "textbox",
|
||||
"value": "John Doe"
|
||||
},
|
||||
{
|
||||
"name": "Email Address",
|
||||
"ref": "e16",
|
||||
"type": "textbox",
|
||||
"value": "[email protected]"
|
||||
},
|
||||
{
|
||||
"name": "Street Address",
|
||||
"ref": "e19",
|
||||
"type": "textbox",
|
||||
"value": "123 Main St"
|
||||
},
|
||||
{
|
||||
"name": "City",
|
||||
"ref": "e22",
|
||||
"type": "textbox",
|
||||
"value": "Anytown"
|
||||
},
|
||||
{
|
||||
"name": "State/Province",
|
||||
"ref": "e25",
|
||||
"type": "textbox",
|
||||
"value": "XYZ"
|
||||
},
|
||||
{
|
||||
"name": "ZIP/Postal Code",
|
||||
"ref": "e28",
|
||||
"type": "textbox",
|
||||
"value": "12345"
|
||||
}
|
||||
]
|
||||
},
|
||||
"id": "tooluse_YGgfR5fAR6OJcd7Q9M4t7w"
|
||||
}
|
||||
]
|
||||
},
|
||||
"usage": {
|
||||
"input": 7601,
|
||||
"output": 298
|
||||
}
|
||||
},
|
||||
"faae385281a92076a48da71d391fa0e477894e6d": {
|
||||
"result": {
|
||||
"role": "assistant",
|
||||
"content": "",
|
||||
"toolCalls": [
|
||||
{
|
||||
"name": "browser_click",
|
||||
"arguments": {
|
||||
"element": "Order button",
|
||||
"ref": "e29"
|
||||
},
|
||||
"id": "tooluse_qo1cC8s9SB-sPPPgTk36sA"
|
||||
}
|
||||
]
|
||||
},
|
||||
"usage": {
|
||||
"input": 8123,
|
||||
"output": 72
|
||||
}
|
||||
},
|
||||
"eaab8ebfa4b63c74ba919e849ba983fcb874528f": {
|
||||
"result": {
|
||||
"role": "assistant",
|
||||
"content": "",
|
||||
"toolCalls": [
|
||||
{
|
||||
"name": "browser_handle_dialog",
|
||||
"arguments": {
|
||||
"accept": true
|
||||
},
|
||||
"id": "tooluse_EWFnsSPoTY6i4Qf7bpEwjg"
|
||||
}
|
||||
]
|
||||
},
|
||||
"usage": {
|
||||
"input": 8274,
|
||||
"output": 55
|
||||
}
|
||||
},
|
||||
"2c31a0c54acc5a4c4f9b8e4c7acdbf94a81eebbb": {
|
||||
"result": {
|
||||
"role": "assistant",
|
||||
"content": "",
|
||||
"toolCalls": [
|
||||
{
|
||||
"name": "report_result",
|
||||
"arguments": {
|
||||
"deliveryDays": 3,
|
||||
"price": 653.96
|
||||
},
|
||||
"id": "tooluse_I1b-G92TS7qIuUsLtl020w"
|
||||
}
|
||||
]
|
||||
},
|
||||
"usage": {
|
||||
"input": 9136,
|
||||
"output": 75
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,492 @@
|
||||
{
|
||||
"facb950333d3cbebad4cb469c846c6a03a72dce8": {
|
||||
"result": {
|
||||
"role": "assistant",
|
||||
"content": "I'll help you navigate to the form, fill it out with the specified details, and complete the order. Let me start by navigating to the page.",
|
||||
"toolCalls": []
|
||||
},
|
||||
"usage": {
|
||||
"input": 4484,
|
||||
"output": 126
|
||||
}
|
||||
},
|
||||
"f65dd22f0f79ffd6398d42230797a4a53c72c9ab": {
|
||||
"result": {
|
||||
"role": "assistant",
|
||||
"content": "Navigate to the form page to start the ordering process",
|
||||
"toolCalls": [
|
||||
{
|
||||
"name": "browser_navigate",
|
||||
"arguments": {
|
||||
"url": "http://localhost:<PORT>/evals/fill-form.html"
|
||||
},
|
||||
"id": "tooluse_BCc-AvDnQWmc8O81Stj2YQ"
|
||||
}
|
||||
]
|
||||
},
|
||||
"usage": {
|
||||
"input": 4540,
|
||||
"output": 93
|
||||
}
|
||||
},
|
||||
"eb63c3301417583c538a6501c36797d42e161a1a": {
|
||||
"result": {
|
||||
"role": "assistant",
|
||||
"content": "Take a snapshot to see the current page state",
|
||||
"toolCalls": [
|
||||
{
|
||||
"name": "browser_snapshot",
|
||||
"arguments": {},
|
||||
"id": "tooluse_EQPPTsKBT_G_YsVMHNZnAw"
|
||||
}
|
||||
]
|
||||
},
|
||||
"usage": {
|
||||
"input": 5242,
|
||||
"output": 61
|
||||
}
|
||||
},
|
||||
"190598029fe374115a87b39e2d631689f712f8c5": {
|
||||
"result": {
|
||||
"role": "assistant",
|
||||
"content": "Fill out the form for the first item - blue table",
|
||||
"toolCalls": []
|
||||
},
|
||||
"usage": {
|
||||
"input": 5878,
|
||||
"output": 350
|
||||
}
|
||||
},
|
||||
"73845baea403b9e42331b19667593fbf028c3429": {
|
||||
"result": {
|
||||
"role": "assistant",
|
||||
"content": "Select Table as the product type",
|
||||
"toolCalls": [
|
||||
{
|
||||
"name": "browser_select_option",
|
||||
"arguments": {
|
||||
"element": "Product Type dropdown",
|
||||
"ref": "e7",
|
||||
"values": [
|
||||
"Table"
|
||||
]
|
||||
},
|
||||
"id": "tooluse_WVHv7CI3R-WqyOj6O7LTKg"
|
||||
}
|
||||
]
|
||||
},
|
||||
"usage": {
|
||||
"input": 5913,
|
||||
"output": 116
|
||||
}
|
||||
},
|
||||
"5cac5e0e4742f5674f0c4c7fd933d7e4c84f0513": {
|
||||
"result": {
|
||||
"role": "assistant",
|
||||
"content": "Select Blue as the color",
|
||||
"toolCalls": []
|
||||
},
|
||||
"usage": {
|
||||
"input": 6165,
|
||||
"output": 99
|
||||
}
|
||||
},
|
||||
"67a472d6a9e6e9c965eaa82a70a4bdbb7485eca6": {
|
||||
"result": {
|
||||
"role": "assistant",
|
||||
"content": "",
|
||||
"toolCalls": [
|
||||
{
|
||||
"name": "browser_select_option",
|
||||
"arguments": {
|
||||
"element": "Color dropdown",
|
||||
"ref": "e10",
|
||||
"values": [
|
||||
"Blue"
|
||||
]
|
||||
},
|
||||
"id": "tooluse_f86G0fVES260pD2OGjYdCA"
|
||||
}
|
||||
]
|
||||
},
|
||||
"usage": {
|
||||
"input": 6194,
|
||||
"output": 93
|
||||
}
|
||||
},
|
||||
"65f822bec8f70205e0d58e9a2ee09a3707ed6401": {
|
||||
"result": {
|
||||
"role": "assistant",
|
||||
"content": "Fill in customer information for the order",
|
||||
"toolCalls": [
|
||||
{
|
||||
"name": "browser_fill_form",
|
||||
"arguments": {
|
||||
"fields": [
|
||||
{
|
||||
"name": "Full Name",
|
||||
"ref": "e13",
|
||||
"type": "textbox",
|
||||
"value": "John Doe"
|
||||
},
|
||||
{
|
||||
"name": "Email Address",
|
||||
"ref": "e16",
|
||||
"type": "textbox",
|
||||
"value": "[email protected]"
|
||||
},
|
||||
{
|
||||
"name": "Street Address",
|
||||
"ref": "e19",
|
||||
"type": "textbox",
|
||||
"value": "123 Main St"
|
||||
},
|
||||
{
|
||||
"name": "City",
|
||||
"ref": "e22",
|
||||
"type": "textbox",
|
||||
"value": "Anytown"
|
||||
},
|
||||
{
|
||||
"name": "State/Province",
|
||||
"ref": "e25",
|
||||
"type": "textbox",
|
||||
"value": "XYZ"
|
||||
},
|
||||
{
|
||||
"name": "ZIP/Postal Code",
|
||||
"ref": "e28",
|
||||
"type": "textbox",
|
||||
"value": "12345"
|
||||
}
|
||||
]
|
||||
},
|
||||
"id": "tooluse_ztCsxsUlTueuOPgHsO83dA"
|
||||
}
|
||||
]
|
||||
},
|
||||
"usage": {
|
||||
"input": 6441,
|
||||
"output": 343
|
||||
}
|
||||
},
|
||||
"6b1f789361769bb90eb776cff6df695394e90ff5": {
|
||||
"result": {
|
||||
"role": "assistant",
|
||||
"content": "Click the Order button to submit the first order",
|
||||
"toolCalls": []
|
||||
},
|
||||
"usage": {
|
||||
"input": 6879,
|
||||
"output": 82
|
||||
}
|
||||
},
|
||||
"f036e1c5e71f07be6729fc6fc0ef5a7514564633": {
|
||||
"result": {
|
||||
"role": "assistant",
|
||||
"content": "",
|
||||
"toolCalls": [
|
||||
{
|
||||
"name": "browser_click",
|
||||
"arguments": {
|
||||
"element": "Order button",
|
||||
"ref": "e29"
|
||||
},
|
||||
"id": "tooluse_7Bw_du6XShCs0VM46yds3A"
|
||||
}
|
||||
]
|
||||
},
|
||||
"usage": {
|
||||
"input": 6912,
|
||||
"output": 72
|
||||
}
|
||||
},
|
||||
"44e579cf6dfa27d84426d8afbffda3bb20e978b5": {
|
||||
"result": {
|
||||
"role": "assistant",
|
||||
"content": "Accept the order confirmation dialog",
|
||||
"toolCalls": []
|
||||
},
|
||||
"usage": {
|
||||
"input": 7064,
|
||||
"output": 61
|
||||
}
|
||||
},
|
||||
"b1043d04c5761a4273d3c6d358e18727ae53bedd": {
|
||||
"result": {
|
||||
"role": "assistant",
|
||||
"content": "",
|
||||
"toolCalls": [
|
||||
{
|
||||
"name": "browser_handle_dialog",
|
||||
"arguments": {
|
||||
"accept": true
|
||||
},
|
||||
"id": "tooluse_CBKRGvaMT8-0bxI3LmpgLQ"
|
||||
}
|
||||
]
|
||||
},
|
||||
"usage": {
|
||||
"input": 7093,
|
||||
"output": 55
|
||||
}
|
||||
},
|
||||
"4a20e94c802689f0e5795cc01d0754593c95b8d9": {
|
||||
"result": {
|
||||
"role": "assistant",
|
||||
"content": "Click Place Another Order button to order the second item",
|
||||
"toolCalls": []
|
||||
},
|
||||
"usage": {
|
||||
"input": 7955,
|
||||
"output": 85
|
||||
}
|
||||
},
|
||||
"02d0f1acdf58e14bfacf4fb3d1a04973726b3f6a": {
|
||||
"result": {
|
||||
"role": "assistant",
|
||||
"content": "",
|
||||
"toolCalls": [
|
||||
{
|
||||
"name": "browser_click",
|
||||
"arguments": {
|
||||
"element": "Place Another Order button",
|
||||
"ref": "e58"
|
||||
},
|
||||
"id": "tooluse_mEVX8xrBQ-u7BPjnZ7PPKQ"
|
||||
}
|
||||
]
|
||||
},
|
||||
"usage": {
|
||||
"input": 7989,
|
||||
"output": 74
|
||||
}
|
||||
},
|
||||
"19504162e60fbdd96072af88bd669b3e88860524": {
|
||||
"result": {
|
||||
"role": "assistant",
|
||||
"content": "Select Desk as the product type for the second order",
|
||||
"toolCalls": []
|
||||
},
|
||||
"usage": {
|
||||
"input": 8688,
|
||||
"output": 107
|
||||
}
|
||||
},
|
||||
"302dcb0c44750d90fdf69073db4a6533d657af9c": {
|
||||
"result": {
|
||||
"role": "assistant",
|
||||
"content": "",
|
||||
"toolCalls": [
|
||||
{
|
||||
"name": "browser_select_option",
|
||||
"arguments": {
|
||||
"element": "Product Type dropdown",
|
||||
"ref": "e7",
|
||||
"values": [
|
||||
"Desk"
|
||||
]
|
||||
},
|
||||
"id": "tooluse_BaMRqzC-QbeyPBC-9RP0aw"
|
||||
}
|
||||
]
|
||||
},
|
||||
"usage": {
|
||||
"input": 8723,
|
||||
"output": 95
|
||||
}
|
||||
},
|
||||
"b40f9a90caa41102d3b795a22a005ad9042b659e": {
|
||||
"result": {
|
||||
"role": "assistant",
|
||||
"content": "Select Green as the color for the desk",
|
||||
"toolCalls": []
|
||||
},
|
||||
"usage": {
|
||||
"input": 8971,
|
||||
"output": 11
|
||||
}
|
||||
},
|
||||
"9c502c32a6f3797f941b683c19473c89cec2a1b4": {
|
||||
"result": {
|
||||
"role": "assistant",
|
||||
"content": "",
|
||||
"toolCalls": [
|
||||
{
|
||||
"name": "browser_select_option",
|
||||
"arguments": {
|
||||
"element": "Color dropdown",
|
||||
"ref": "e10",
|
||||
"values": [
|
||||
"Green"
|
||||
]
|
||||
},
|
||||
"id": "tooluse_bvxZZXldSu28CR0zYOoIng"
|
||||
}
|
||||
]
|
||||
},
|
||||
"usage": {
|
||||
"input": 9003,
|
||||
"output": 93
|
||||
}
|
||||
},
|
||||
"ce3d6682dda9339c3efd384e5bd23e9aef4852fc": {
|
||||
"result": {
|
||||
"role": "assistant",
|
||||
"content": "Fill in the customer information for the second order",
|
||||
"toolCalls": []
|
||||
},
|
||||
"usage": {
|
||||
"input": 9250,
|
||||
"output": 250
|
||||
}
|
||||
},
|
||||
"92858cba54b40fc65bc94e7f1ffd16f427f3f1b0": {
|
||||
"result": {
|
||||
"role": "assistant",
|
||||
"content": "",
|
||||
"toolCalls": [
|
||||
{
|
||||
"name": "browser_fill_form",
|
||||
"arguments": {
|
||||
"fields": [
|
||||
{
|
||||
"name": "Full Name",
|
||||
"ref": "e13",
|
||||
"type": "textbox",
|
||||
"value": "John Doe"
|
||||
},
|
||||
{
|
||||
"name": "Email Address",
|
||||
"ref": "e16",
|
||||
"type": "textbox",
|
||||
"value": "[email protected]"
|
||||
},
|
||||
{
|
||||
"name": "Street Address",
|
||||
"ref": "e19",
|
||||
"type": "textbox",
|
||||
"value": "123 Main St"
|
||||
},
|
||||
{
|
||||
"name": "City",
|
||||
"ref": "e22",
|
||||
"type": "textbox",
|
||||
"value": "Anytown"
|
||||
},
|
||||
{
|
||||
"name": "State/Province",
|
||||
"ref": "e25",
|
||||
"type": "textbox",
|
||||
"value": "XYZ"
|
||||
},
|
||||
{
|
||||
"name": "ZIP/Postal Code",
|
||||
"ref": "e28",
|
||||
"type": "textbox",
|
||||
"value": "12345"
|
||||
}
|
||||
]
|
||||
},
|
||||
"id": "tooluse_MiTMyTOaRoCmK0EapkxU7g"
|
||||
}
|
||||
]
|
||||
},
|
||||
"usage": {
|
||||
"input": 9283,
|
||||
"output": 240
|
||||
}
|
||||
},
|
||||
"3c11d68c47519b7364a7dd5c1be26d6b7cd16a66": {
|
||||
"result": {
|
||||
"role": "assistant",
|
||||
"content": "Click the Order button to submit the second order",
|
||||
"toolCalls": []
|
||||
},
|
||||
"usage": {
|
||||
"input": 9714,
|
||||
"output": 12
|
||||
}
|
||||
},
|
||||
"334effd07d305facef120ed49d47eca9546b9ed2": {
|
||||
"result": {
|
||||
"role": "assistant",
|
||||
"content": "",
|
||||
"toolCalls": [
|
||||
{
|
||||
"name": "browser_click",
|
||||
"arguments": {
|
||||
"element": "Order button",
|
||||
"ref": "e29"
|
||||
},
|
||||
"id": "tooluse_jana5W7FSLiGM40Y58vxuA"
|
||||
}
|
||||
]
|
||||
},
|
||||
"usage": {
|
||||
"input": 9747,
|
||||
"output": 72
|
||||
}
|
||||
},
|
||||
"3e70d82f485d3bb257053e605a888e391d8458f3": {
|
||||
"result": {
|
||||
"role": "assistant",
|
||||
"content": "Accept the order confirmation dialog for the second order",
|
||||
"toolCalls": []
|
||||
},
|
||||
"usage": {
|
||||
"input": 9899,
|
||||
"output": 12
|
||||
}
|
||||
},
|
||||
"f6df99806ce1301aedf58fd2ebfc77b3645f849b": {
|
||||
"result": {
|
||||
"role": "assistant",
|
||||
"content": "",
|
||||
"toolCalls": [
|
||||
{
|
||||
"name": "browser_handle_dialog",
|
||||
"arguments": {
|
||||
"accept": true
|
||||
},
|
||||
"id": "tooluse_j3MT0SvtSBuucN6YXXxfZg"
|
||||
}
|
||||
]
|
||||
},
|
||||
"usage": {
|
||||
"input": 9932,
|
||||
"output": 55
|
||||
}
|
||||
},
|
||||
"c6d1b1ddc05aa572b48a0abfebaf84117d38aefd": {
|
||||
"result": {
|
||||
"role": "assistant",
|
||||
"content": "Report the total price and maximum estimated delivery time for both orders",
|
||||
"toolCalls": []
|
||||
},
|
||||
"usage": {
|
||||
"input": 10795,
|
||||
"output": 88
|
||||
}
|
||||
},
|
||||
"b2b70eb2a30bcd06f1cd7819b9a7d4e275a528b6": {
|
||||
"result": {
|
||||
"role": "assistant",
|
||||
"content": "",
|
||||
"toolCalls": [
|
||||
{
|
||||
"name": "report_result",
|
||||
"arguments": {
|
||||
"deliveryDays": 3,
|
||||
"price": 653.96
|
||||
},
|
||||
"id": "tooluse_CR5rFd09QRGnH5-Lt3-pGw"
|
||||
}
|
||||
]
|
||||
},
|
||||
"usage": {
|
||||
"input": 10831,
|
||||
"output": 75
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,404 @@
|
||||
{
|
||||
"facb950333d3cbebad4cb469c846c6a03a72dce8": {
|
||||
"result": {
|
||||
"role": "assistant",
|
||||
"content": "I'll help you navigate to the form, fill it out with the specified details, and report back the results. Let me start by navigating to the page.",
|
||||
"toolCalls": []
|
||||
},
|
||||
"usage": {
|
||||
"input": 4484,
|
||||
"output": 136
|
||||
}
|
||||
},
|
||||
"e9daf936478b73139df8f3ea61bc82a1a13c3cdb": {
|
||||
"result": {
|
||||
"role": "assistant",
|
||||
"content": "Navigate to the form page",
|
||||
"toolCalls": [
|
||||
{
|
||||
"name": "browser_navigate",
|
||||
"arguments": {
|
||||
"url": "http://localhost:<PORT>/evals/fill-form.html"
|
||||
},
|
||||
"id": "tooluse_2QTlFMmcQlqV0a1rVultZw"
|
||||
}
|
||||
]
|
||||
},
|
||||
"usage": {
|
||||
"input": 4541,
|
||||
"output": 88
|
||||
}
|
||||
},
|
||||
"7dc1192eb36584ccdef322c804036a775f23f7a2": {
|
||||
"result": {
|
||||
"role": "assistant",
|
||||
"content": "Check the current state of the form before filling",
|
||||
"toolCalls": [
|
||||
{
|
||||
"name": "browser_snapshot",
|
||||
"arguments": {},
|
||||
"id": "tooluse_Xq-IYmyLTua6t1KRVizOew"
|
||||
}
|
||||
]
|
||||
},
|
||||
"usage": {
|
||||
"input": 5238,
|
||||
"output": 61
|
||||
}
|
||||
},
|
||||
"e960fd42b63a75629d10e2b064a712cf4bb222c8": {
|
||||
"result": {
|
||||
"role": "assistant",
|
||||
"content": "Fill the first order form for a blue table",
|
||||
"toolCalls": []
|
||||
},
|
||||
"usage": {
|
||||
"input": 5874,
|
||||
"output": 126
|
||||
}
|
||||
},
|
||||
"ab8368051e5eb0335afc2627359409ff6e527b22": {
|
||||
"result": {
|
||||
"role": "assistant",
|
||||
"content": "Select Table as the product type for the first order",
|
||||
"toolCalls": [
|
||||
{
|
||||
"name": "browser_select_option",
|
||||
"arguments": {
|
||||
"element": "Product Type dropdown",
|
||||
"ref": "e7",
|
||||
"values": [
|
||||
"Table"
|
||||
]
|
||||
},
|
||||
"id": "tooluse_qDJgPasgTZazcKmtzotHZg"
|
||||
}
|
||||
]
|
||||
},
|
||||
"usage": {
|
||||
"input": 5907,
|
||||
"output": 120
|
||||
}
|
||||
},
|
||||
"eefa023ed6d14b88656c764ec83bf98bee091d0a": {
|
||||
"result": {
|
||||
"role": "assistant",
|
||||
"content": "Select Blue as the color",
|
||||
"toolCalls": []
|
||||
},
|
||||
"usage": {
|
||||
"input": 6163,
|
||||
"output": 99
|
||||
}
|
||||
},
|
||||
"c7ddf8898251a2d2a7fcadf36e226a14f2d32955": {
|
||||
"result": {
|
||||
"role": "assistant",
|
||||
"content": "",
|
||||
"toolCalls": [
|
||||
{
|
||||
"name": "browser_select_option",
|
||||
"arguments": {
|
||||
"element": "Color dropdown",
|
||||
"ref": "e10",
|
||||
"values": [
|
||||
"Blue"
|
||||
]
|
||||
},
|
||||
"id": "tooluse_K_TcaZuDTaaPDZT-qhxH0g"
|
||||
}
|
||||
]
|
||||
},
|
||||
"usage": {
|
||||
"input": 6192,
|
||||
"output": 93
|
||||
}
|
||||
},
|
||||
"5ccb74547a127df0533ca011b1ac02fb9df840c3": {
|
||||
"result": {
|
||||
"role": "assistant",
|
||||
"content": "Fill in all the customer details in the form",
|
||||
"toolCalls": [
|
||||
{
|
||||
"name": "browser_fill_form",
|
||||
"arguments": {
|
||||
"fields": [
|
||||
{
|
||||
"name": "Full Name",
|
||||
"ref": "e13",
|
||||
"type": "textbox",
|
||||
"value": "John Doe"
|
||||
},
|
||||
{
|
||||
"name": "Email Address",
|
||||
"ref": "e16",
|
||||
"type": "textbox",
|
||||
"value": "[email protected]"
|
||||
},
|
||||
{
|
||||
"name": "Street Address",
|
||||
"ref": "e19",
|
||||
"type": "textbox",
|
||||
"value": "123 Main St"
|
||||
},
|
||||
{
|
||||
"name": "City",
|
||||
"ref": "e22",
|
||||
"type": "textbox",
|
||||
"value": "Anytown"
|
||||
},
|
||||
{
|
||||
"name": "State/Province",
|
||||
"ref": "e25",
|
||||
"type": "textbox",
|
||||
"value": "XYZ"
|
||||
},
|
||||
{
|
||||
"name": "ZIP/Postal Code",
|
||||
"ref": "e28",
|
||||
"type": "textbox",
|
||||
"value": "12345"
|
||||
}
|
||||
]
|
||||
},
|
||||
"id": "tooluse_fevjVnF0S2OhEUg_pdQNog"
|
||||
}
|
||||
]
|
||||
},
|
||||
"usage": {
|
||||
"input": 6439,
|
||||
"output": 345
|
||||
}
|
||||
},
|
||||
"1ef5b1de2508c7962e6e4b4dcefe59c11179680d": {
|
||||
"result": {
|
||||
"role": "assistant",
|
||||
"content": "Submit the order form",
|
||||
"toolCalls": []
|
||||
},
|
||||
"usage": {
|
||||
"input": 6879,
|
||||
"output": 77
|
||||
}
|
||||
},
|
||||
"f096f02c459d6fa500047ad6ba5e65c0e5222e20": {
|
||||
"result": {
|
||||
"role": "assistant",
|
||||
"content": "",
|
||||
"toolCalls": [
|
||||
{
|
||||
"name": "browser_click",
|
||||
"arguments": {
|
||||
"element": "Order button",
|
||||
"ref": "e29"
|
||||
},
|
||||
"id": "tooluse_cDuf7VYITsOk3SPizGkdbw"
|
||||
}
|
||||
]
|
||||
},
|
||||
"usage": {
|
||||
"input": 6907,
|
||||
"output": 72
|
||||
}
|
||||
},
|
||||
"a28b9ead7a286733ddbf6c52fd0c3ed1d5954725": {
|
||||
"result": {
|
||||
"role": "assistant",
|
||||
"content": "",
|
||||
"toolCalls": [
|
||||
{
|
||||
"name": "browser_handle_dialog",
|
||||
"arguments": {
|
||||
"accept": true
|
||||
},
|
||||
"id": "tooluse_A0g51QouRsWL9CsOTx5Ycg"
|
||||
}
|
||||
]
|
||||
},
|
||||
"usage": {
|
||||
"input": 7059,
|
||||
"output": 55
|
||||
}
|
||||
},
|
||||
"fb6f4514207d7e27fa9a5c6da79320dd6d5ab572": {
|
||||
"result": {
|
||||
"role": "assistant",
|
||||
"content": "",
|
||||
"toolCalls": [
|
||||
{
|
||||
"name": "browser_click",
|
||||
"arguments": {
|
||||
"element": "Place Another Order button",
|
||||
"ref": "e58"
|
||||
},
|
||||
"id": "tooluse_4U7nwg7ZTMiom22WRl-FJw"
|
||||
}
|
||||
]
|
||||
},
|
||||
"usage": {
|
||||
"input": 7920,
|
||||
"output": 74
|
||||
}
|
||||
},
|
||||
"4a73f42e2d9bdb78c9a2e5390bc7a964fbb343c0": {
|
||||
"result": {
|
||||
"role": "assistant",
|
||||
"content": "",
|
||||
"toolCalls": [
|
||||
{
|
||||
"name": "browser_select_option",
|
||||
"arguments": {
|
||||
"element": "Product Type dropdown",
|
||||
"ref": "e7",
|
||||
"values": [
|
||||
"Desk"
|
||||
]
|
||||
},
|
||||
"id": "tooluse_67fzJrlgS06obb4sPHxk-Q"
|
||||
}
|
||||
]
|
||||
},
|
||||
"usage": {
|
||||
"input": 8618,
|
||||
"output": 95
|
||||
}
|
||||
},
|
||||
"cf8bc3f7c495171c5541706d297582791b7c316f": {
|
||||
"result": {
|
||||
"role": "assistant",
|
||||
"content": "",
|
||||
"toolCalls": [
|
||||
{
|
||||
"name": "browser_select_option",
|
||||
"arguments": {
|
||||
"element": "Color dropdown",
|
||||
"ref": "e10",
|
||||
"values": [
|
||||
"Green"
|
||||
]
|
||||
},
|
||||
"id": "tooluse_CroDBxgYS1akDERsrWsRAg"
|
||||
}
|
||||
]
|
||||
},
|
||||
"usage": {
|
||||
"input": 8865,
|
||||
"output": 93
|
||||
}
|
||||
},
|
||||
"995ae7de03de31d0deaa9f9c415b682386ea0290": {
|
||||
"result": {
|
||||
"role": "assistant",
|
||||
"content": "",
|
||||
"toolCalls": [
|
||||
{
|
||||
"name": "browser_fill_form",
|
||||
"arguments": {
|
||||
"fields": [
|
||||
{
|
||||
"name": "Full Name",
|
||||
"ref": "e13",
|
||||
"type": "textbox",
|
||||
"value": "John Doe"
|
||||
},
|
||||
{
|
||||
"name": "Email Address",
|
||||
"ref": "e16",
|
||||
"type": "textbox",
|
||||
"value": "[email protected]"
|
||||
},
|
||||
{
|
||||
"name": "Street Address",
|
||||
"ref": "e19",
|
||||
"type": "textbox",
|
||||
"value": "123 Main St"
|
||||
},
|
||||
{
|
||||
"name": "City",
|
||||
"ref": "e22",
|
||||
"type": "textbox",
|
||||
"value": "Anytown"
|
||||
},
|
||||
{
|
||||
"name": "State/Province",
|
||||
"ref": "e25",
|
||||
"type": "textbox",
|
||||
"value": "XYZ"
|
||||
},
|
||||
{
|
||||
"name": "ZIP/Postal Code",
|
||||
"ref": "e28",
|
||||
"type": "textbox",
|
||||
"value": "12345"
|
||||
}
|
||||
]
|
||||
},
|
||||
"id": "tooluse_DLFD9lwrQb2Oc_4NpnYOYg"
|
||||
}
|
||||
]
|
||||
},
|
||||
"usage": {
|
||||
"input": 9111,
|
||||
"output": 240
|
||||
}
|
||||
},
|
||||
"6ec90af600847bd4dd2a0fa6b8ea5c08fd0e59d0": {
|
||||
"result": {
|
||||
"role": "assistant",
|
||||
"content": "",
|
||||
"toolCalls": [
|
||||
{
|
||||
"name": "browser_click",
|
||||
"arguments": {
|
||||
"element": "Order button",
|
||||
"ref": "e29"
|
||||
},
|
||||
"id": "tooluse_oNWDYmu7RTClOEuCTrn7Sg"
|
||||
}
|
||||
]
|
||||
},
|
||||
"usage": {
|
||||
"input": 9541,
|
||||
"output": 72
|
||||
}
|
||||
},
|
||||
"c46467b8f90fdfbfdf5081dda99e2055b6f6152f": {
|
||||
"result": {
|
||||
"role": "assistant",
|
||||
"content": "",
|
||||
"toolCalls": [
|
||||
{
|
||||
"name": "browser_handle_dialog",
|
||||
"arguments": {
|
||||
"accept": true
|
||||
},
|
||||
"id": "tooluse_qBV8FnNVTMiBZ8oLuUVYOQ"
|
||||
}
|
||||
]
|
||||
},
|
||||
"usage": {
|
||||
"input": 9692,
|
||||
"output": 55
|
||||
}
|
||||
},
|
||||
"c9ebdba51d456673a83fd1ec4e484fea864ef5b7": {
|
||||
"result": {
|
||||
"role": "assistant",
|
||||
"content": "",
|
||||
"toolCalls": [
|
||||
{
|
||||
"name": "report_result",
|
||||
"arguments": {
|
||||
"deliveryDays": 3,
|
||||
"price": 653.96
|
||||
},
|
||||
"id": "tooluse_Tsf2nC_CR6CMtZL91D0ovA"
|
||||
}
|
||||
]
|
||||
},
|
||||
"usage": {
|
||||
"input": 10554,
|
||||
"output": 75
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
/**
|
||||
* Copyright (c) Microsoft Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import z from 'zod';
|
||||
import { zodToJsonSchema } from 'zod-to-json-schema';
|
||||
|
||||
import { test, expect } from './fixtures';
|
||||
|
||||
import type * as tinyLoop from 'tiny-loop';
|
||||
|
||||
test.use({ mcpHeadless: async ({ headless }, use) => use(headless) });
|
||||
|
||||
test('fill the form', async ({ loop, client, server }) => {
|
||||
expect(await loop.run<{ price: number, deliveryDays: number }>(`
|
||||
Navigate to ${server.PREFIX + '/evals/fill-form.html'} via Playwright MCP.
|
||||
Order a blue table and a green desk.
|
||||
Use the following details for the order form:
|
||||
- Name: John Doe
|
||||
- Address: 123 Main St, Anytown, XYZ state
|
||||
- Zip Code: 12345
|
||||
- Email: [email protected]
|
||||
Report back total price for both items, and maximum estimated delivery.
|
||||
Do not take screenshots.
|
||||
`, {
|
||||
tools: (await client.listTools()).tools as tinyLoop.Tool[],
|
||||
callTool: (params => client.callTool(params)) as tinyLoop.ToolCallback,
|
||||
resultSchema: zodToJsonSchema(z.object({
|
||||
price: z.number().describe('Total price in USD.'),
|
||||
deliveryDays: z.number().describe('Maximum estimated delivery time in days.'),
|
||||
})) as tinyLoop.Schema,
|
||||
})).toEqual({ price: 653.96, deliveryDays: 3 });
|
||||
});
|
||||
@@ -17,6 +17,8 @@
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import { chromium } from 'playwright';
|
||||
import { Loop } from 'tiny-loop';
|
||||
import debug from 'debug';
|
||||
|
||||
import { test as baseTest, expect as baseExpect } from '@playwright/test';
|
||||
import { StdioClientTransport } from '@modelcontextprotocol/sdk/client/stdio.js';
|
||||
@@ -65,6 +67,7 @@ type TestFixtures = {
|
||||
server: TestServer;
|
||||
httpsServer: TestServer;
|
||||
mcpHeadless: boolean;
|
||||
loop: Loop;
|
||||
};
|
||||
|
||||
type WorkerFixtures = {
|
||||
@@ -184,6 +187,28 @@ export const test = serverTest.extend<TestFixtures & TestOptions, WorkerFixtures
|
||||
mcpBrowser: ['chrome', { option: true }],
|
||||
|
||||
mcpServerType: ['mcp', { option: true }],
|
||||
|
||||
loop: async ({ server }, use) => {
|
||||
const provider = 'copilot';
|
||||
const cacheFile = path.join(__dirname, '__cache__', provider, sanitizeFileName(test.info().titlePath.join(' ') + '-repeat' + test.info().repeatEachIndex) + '.json');
|
||||
const dataBefore = await fs.promises.readFile(cacheFile, 'utf-8').catch(() => '{}');
|
||||
let cache = {};
|
||||
try {
|
||||
cache = JSON.parse(dataBefore);
|
||||
} catch {
|
||||
}
|
||||
const caches = { before: cache, after: {} };
|
||||
await use(new Loop(provider, {
|
||||
caches,
|
||||
secrets: { PORT: String(server.PORT) },
|
||||
logger: (category, ...args) => debug(category)(...args),
|
||||
}));
|
||||
const dataAfter = JSON.stringify(caches.after, null, 2);
|
||||
if (test.info().status === 'passed' && dataBefore !== dataAfter) {
|
||||
await fs.promises.mkdir(path.dirname(cacheFile), { recursive: true });
|
||||
await fs.promises.writeFile(cacheFile, dataAfter);
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
async function createTransport(mcpServerType: TestOptions['mcpServerType'], options: { args: string[], env: NodeJS.ProcessEnv, cwd: string }): Promise<{
|
||||
@@ -312,3 +337,7 @@ export async function prepareDebugTest(startClient: StartClient, testFile?: stri
|
||||
const [, id] = listResult.content[0].text.match(/\[id=([^\]]+)\]/);
|
||||
return { client, id };
|
||||
}
|
||||
|
||||
function sanitizeFileName(name: string): string {
|
||||
return name.replace('.ts', '').replace(/[^a-zA-Z0-9_]+/g, '-');
|
||||
}
|
||||
|
||||
@@ -1,32 +0,0 @@
|
||||
/**
|
||||
* Copyright (c) Microsoft Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { defineConfig } from '@playwright/test';
|
||||
|
||||
import type { TestOptions } from './fixtures';
|
||||
|
||||
export default defineConfig<TestOptions>({
|
||||
testDir: './',
|
||||
grep: /extension/,
|
||||
fullyParallel: true,
|
||||
forbidOnly: !!process.env.CI,
|
||||
retries: process.env.CI ? 2 : 0,
|
||||
workers: process.env.CI ? 1 : undefined,
|
||||
reporter: 'list',
|
||||
projects: [
|
||||
{ name: 'chromium', use: { mcpBrowser: 'chromium' } },
|
||||
],
|
||||
});
|
||||
@@ -14,12 +14,15 @@
|
||||
* limitations under the License.
|
||||
*/
|
||||
import * as path from 'path';
|
||||
import dotenv from 'dotenv';
|
||||
|
||||
import { defineConfig } from '@playwright/test';
|
||||
|
||||
import type { TestOptions } from './fixtures';
|
||||
import type { ReporterDescription } from '@playwright/test';
|
||||
|
||||
dotenv.config({ path: path.resolve(__dirname, '../../.env'), quiet: true });
|
||||
|
||||
const rootTestDir = path.join(__dirname, '..');
|
||||
const testDir = path.join(rootTestDir, 'mcp');
|
||||
const outputDir = path.join(__dirname, '..', '..', 'test-results');
|
||||
@@ -44,7 +47,6 @@ const metadata = {
|
||||
|
||||
export default defineConfig<TestOptions>({
|
||||
testDir: rootTestDir,
|
||||
grepInvert: /extension/,
|
||||
fullyParallel: true,
|
||||
forbidOnly: !!process.env.CI,
|
||||
workers: process.env.CI ? 2 : undefined,
|
||||
@@ -55,5 +57,14 @@ export default defineConfig<TestOptions>({
|
||||
{ name: 'firefox', use: { mcpBrowser: 'firefox' }, metadata: { ...metadata, browserName: 'firefox' }, testDir },
|
||||
{ name: 'webkit', use: { mcpBrowser: 'webkit' }, metadata: { ...metadata, browserName: 'webkit' }, testDir },
|
||||
... process.platform === 'win32' ? [{ name: 'msedge', use: { mcpBrowser: 'msedge' }, metadata: { ...metadata, browserName: 'chromium', channel: 'msedge' }, testDir }] : [],
|
||||
{
|
||||
name: 'eval',
|
||||
testDir,
|
||||
testMatch: /.*\.eval\.ts/,
|
||||
repeatEach: 3, // Generate three different trajectories per test.
|
||||
timeout: process.env.CI ? 30_000 : 180_000, // CI should be cache-only, but local use of LLM is slow.
|
||||
use: { mcpBrowser: 'chromium' },
|
||||
metadata: { ...metadata, browserName: 'chromium' },
|
||||
}
|
||||
],
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user