diff --git a/01-features/06-observe-evaluate-optimize-your-agent/02-evaluate/README.md b/01-features/06-observe-evaluate-optimize-your-agent/02-evaluate/README.md index f6621db8..37b927e7 100644 --- a/01-features/06-observe-evaluate-optimize-your-agent/02-evaluate/README.md +++ b/01-features/06-observe-evaluate-optimize-your-agent/02-evaluate/README.md @@ -25,11 +25,13 @@ Test your agents during development and deployment using the on-demand evaluatio Run synchronous, on-demand evaluations using built-in and custom metrics on individual traces. The system uses OpenTelemetry (OTEL) traces to perform scoring and returns a response that includes: + - Score value - Explanation for the score - Token usage **When to use on-demand evaluations:** + - Investigating specific customer interactions or reported issues - Validating fixes for identified problems - Analyzing historical data for quality improvements @@ -43,11 +45,13 @@ The system uses OpenTelemetry (OTEL) traces to perform scoring and returns a res In production, you need continuous performance monitoring across all interactions without manually evaluating each trace. A statistical sample is often sufficient for generating meaningful performance metrics. AgentCore evaluations' online capabilities enable automatic sampling and evaluation: + - Define your sample size and trace selection criteria - Choose your evaluation metrics (built-in or custom) - AgentCore evaluations handles the rest, generating the performance data you need to monitor your agent at scale **When to use online evaluations:** + - Monitoring production agent performance continuously - Catching quality regressions before they impact users - Identifying patterns in user interactions at scale @@ -67,6 +71,7 @@ Both evaluation types rely on **AgentCore observability** to capture agent behav AgentCore relies on **AWS Distro for OpenTelemetry (ADOT)** to instrument different types of OTEL traces across various agent frameworks: **For AgentCore runtime-hosted agents:** + - Instrumentation is automatic with minimal configuration - Simply include `aws-opentelemetry-distro` in your `requirements.txt` - AgentCore runtime handles OTEL configuration automatically @@ -74,21 +79,21 @@ AgentCore relies on **AWS Distro for OpenTelemetry (ADOT)** to instrument differ ## Built-in Evaluators -| Evaluator | Level | Needs Ground Truth | Description | -|:----------|:------|:-------------------|:------------| -| `Builtin.Correctness` | TRACE | `expected_response` | Evaluates whether the information in the agent's response is factually accurate | -| `Builtin.Faithfulness` | TRACE | None | Evaluates whether information in the response is supported by provided context/sources | -| `Builtin.Helpfulness` | TRACE | None | Evaluates from user's perspective how useful and valuable the agent's response is | -| `Builtin.ResponseRelevance` | TRACE | None | Evaluates whether the response appropriately addresses the user's query | -| `Builtin.Conciseness` | TRACE | None | Evaluates whether the response is appropriately brief without missing key information | -| `Builtin.Coherence` | TRACE | None | Evaluates whether the response is logically structured and coherent | -| `Builtin.InstructionFollowing` | TRACE | None | Measures how well the agent follows the provided system instructions | -| `Builtin.Refusal` | TRACE | None | Detects when agent evades questions or directly refuses to answer | -| `Builtin.GoalSuccessRate` | SESSION | `assertions` | Evaluates whether the conversation successfully meets the user's goals | -| `Builtin.ToolSelectionAccuracy` | SESSION | None | Evaluates whether the agent selected the appropriate tool for the task | -| `Builtin.ToolParameterAccuracy` | SESSION | None | Evaluates how accurately the agent extracts parameters from user queries | -| `Builtin.Harmfulness` | TRACE | None | Evaluates whether the response contains harmful content | -| `Builtin.Stereotyping` | TRACE | None | Detects content that makes generalizations about individuals or groups | +| Evaluator | Level | Needs Ground Truth | Description | +| :------------------------------ | :------ | :------------------ | :------------------------------------------------------------------------------------- | +| `Builtin.Correctness` | TRACE | `expected_response` | Evaluates whether the information in the agent's response is factually accurate | +| `Builtin.Faithfulness` | TRACE | None | Evaluates whether information in the response is supported by provided context/sources | +| `Builtin.Helpfulness` | TRACE | None | Evaluates from user's perspective how useful and valuable the agent's response is | +| `Builtin.ResponseRelevance` | TRACE | None | Evaluates whether the response appropriately addresses the user's query | +| `Builtin.Conciseness` | TRACE | None | Evaluates whether the response is appropriately brief without missing key information | +| `Builtin.Coherence` | TRACE | None | Evaluates whether the response is logically structured and coherent | +| `Builtin.InstructionFollowing` | TRACE | None | Measures how well the agent follows the provided system instructions | +| `Builtin.Refusal` | TRACE | None | Detects when agent evades questions or directly refuses to answer | +| `Builtin.GoalSuccessRate` | SESSION | `assertions` | Evaluates whether the conversation successfully meets the user's goals | +| `Builtin.ToolSelectionAccuracy` | SESSION | None | Evaluates whether the agent selected the appropriate tool for the task | +| `Builtin.ToolParameterAccuracy` | SESSION | None | Evaluates how accurately the agent extracts parameters from user queries | +| `Builtin.Harmfulness` | TRACE | None | Evaluates whether the response contains harmful content | +| `Builtin.Stereotyping` | TRACE | None | Detects content that makes generalizations about individuals or groups | **TRACE** evaluators produce one score per conversational turn. **SESSION** evaluators produce one score per complete conversation. @@ -99,21 +104,22 @@ AgentCore relies on **AWS Distro for OpenTelemetry (ADOT)** to instrument differ Three evaluation interfaces are available depending on your use case: -| Interface | Best For | How it runs | -|:----------|:---------|:------------| -| `EvaluationClient` | Ad-hoc debugging, CI spot-checks on known sessions | Client-side, synchronous per session | -| `OnDemandEvaluationDatasetRunner` | Regression testing, CI/CD pipelines with a dataset | Client-side, runs agent + evaluates per scenario | -| `BatchEvaluationRunner` | Baseline snapshots, large-scale evaluation, pre/post comparison | Service-side, aggregate scores per evaluator | +| Interface | Best For | How it runs | +| :-------------------------------- | :-------------------------------------------------------------- | :----------------------------------------------- | +| `EvaluationClient` | Ad-hoc debugging, CI spot-checks on known sessions | Client-side, synchronous per session | +| `OnDemandEvaluationDatasetRunner` | Regression testing, CI/CD pipelines with a dataset | Client-side, runs agent + evaluates per scenario | +| `BatchEvaluationRunner` | Baseline snapshots, large-scale evaluation, pre/post comparison | Service-side, aggregate scores per evaluator | ## evaluation Samples -| Sample | What it demonstrates | -|:-------|:--------------------| -| [`ground-truth-based-evaluation/`](ground-truth-based-evaluation/) | EvaluationClient + DatasetRunner + BatchRunner with expected responses, expected tool trajectories, and session assertions | -| [`llm-as-a-judge-evaluation/`](llm-as-a-judge-evaluation/) | Custom LLM-as-a-judge evaluators (TRACE + SESSION) with ground-truth placeholders alongside built-in evaluators | -| [`custom-code-based-evaluation/`](custom-code-based-evaluation/) | Lambda-backed deterministic evaluators (code-based) for exact data validation, mixed with built-in LLM evaluators; on-demand and online modes | +| Sample | What it demonstrates | +| :----------------------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| [`ground-truth-based-evaluation/`](ground-truth-based-evaluation/) | EvaluationClient + DatasetRunner + BatchRunner with expected responses, expected tool trajectories, and session assertions | +| [`llm-as-a-judge-evaluation/`](llm-as-a-judge-evaluation/) | Custom LLM-as-a-judge evaluators (TRACE + SESSION) with ground-truth placeholders alongside built-in evaluators | +| [`custom-code-based-evaluation/`](custom-code-based-evaluation/) | Lambda-backed deterministic evaluators (code-based) for exact data validation, mixed with built-in LLM evaluators; on-demand and online modes | +| [`supported-frameworks/`](supported-frameworks/) | The same HR Assistant re-implemented in other supported frameworks (OpenAI Agents SDK, LlamaIndex), each deployed and evaluated with built-in and custom evaluators | -All samples share the same HR Assistant agent deployed from `utils/`. +The `ground-truth-based-evaluation/`, `llm-as-a-judge-evaluation/`, and `custom-code-based-evaluation/` samples share the same HR Assistant agent deployed from `utils/`. The `supported-frameworks/` samples re-implement that agent in each framework and deploy it from their own folders. ## Agent Architecture diff --git a/01-features/06-observe-evaluate-optimize-your-agent/02-evaluate/supported-frameworks/README.md b/01-features/06-observe-evaluate-optimize-your-agent/02-evaluate/supported-frameworks/README.md new file mode 100644 index 00000000..5e11cd11 --- /dev/null +++ b/01-features/06-observe-evaluate-optimize-your-agent/02-evaluate/supported-frameworks/README.md @@ -0,0 +1,76 @@ +# Evaluate agents across supported frameworks + +Amazon Bedrock AgentCore Evaluations works with agents built on a range of supported agent frameworks. This folder provides runnable examples for OpenAI Agents SDK and LlamaIndex, showing how the same evaluation flow applies regardless of how the agent is built. + +Each sample deploys the same HR Assistant, the agent used across the sibling `02-evaluate/` samples, re-implemented in its framework. It then evaluates the agent with built-in and custom LLM-as-a-judge evaluators in on-demand and online modes. Because every sample uses the same 5 tools, mock data, system prompt, and ground-truth turns, results are directly comparable across frameworks. + +## Samples + +| Framework | Instrumentation (OpenTelemetry) | LLM | Sample | +| :---------------- | :-------------------------------------------- | :------------------------------------------- | :--------------------------------- | +| OpenAI Agents SDK | `opentelemetry-instrumentation-openai-agents` | OpenAI GPT-5.5 on Bedrock (`openai.gpt-5.5`) | [`openai-agents/`](openai-agents/) | +| LlamaIndex | `opentelemetry-instrumentation-llamaindex` | Amazon Bedrock (`us.amazon.nova-lite-v1:0`) | [`llamaindex/`](llamaindex/) | + +These samples use OpenTelemetry instrumentation. AgentCore Evaluations also supports OpenInference for framework and instrumentation-library pairs listed in [Supported agent frameworks](https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/supported-frameworks.html). On AgentCore Runtime, AWS Distro for OpenTelemetry (ADOT) discovers the installed instrumentation library at startup, so no explicit instrumentation code is needed in the agent. + +The examples in this folder are not the boundary of evaluation support. AgentCore Evaluations uses each span's `scope.name` and framework-specific attributes to classify agent, model, and tool operations. Any framework and instrumentation-library pair in the supported matrix can use the same evaluation APIs when it emits the documented OpenTelemetry or OpenInference schema, including correlated event records when required. A custom scope is not automatically supported solely because its data is transported with OTLP. + +## The shared HR Assistant scenario + +Every sample re-implements the same agent: an HR assistant with 5 tools that return deterministic mock data (PTO balances, HR policies, benefits, pay stubs). Reusing one agent domain keeps the focus on the framework integration rather than the agent itself. Because the data is deterministic, the same evaluation ground truth (`expected_response`, `expected_trajectory`, `assertions`) is valid for every framework, so evaluation scores reflect the framework's behavior rather than differences in the agent's task. + +## Making a supported framework agent evaluable + +The recipe these samples follow generalizes to every supported framework: + +1. Add the instrumentation package for your framework to the deployment dependencies. The evaluation service identifies spans by their `scope.name`, so the package and its declared framework dependency must be importable at runtime. ADOT silently skips an instrumentor whose dependency check fails. +2. Structure the agent so its telemetry is recoverable. Each framework page in the [developer guide](https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/supported-frameworks.html) lists best practices. For example, LlamaIndex agents must be workflow agents (`FunctionAgent` or `ReActAgent`), and OpenAI Agents must keep SDK tracing enabled because the instrumentation hooks into it. +3. Flush telemetry before returning. AgentCore Runtime suspends the microVM between invocations; call `force_flush()` on the tracer and logger providers at the end of each invocation so buffered spans and event records (which carry the response text evaluators score) are not lost. +4. Verify the spans, then evaluate. After invoking the agent, confirm records with your framework's scope name appear in CloudWatch (`aws/spans` and the runtime log group). If `Evaluate` returns "no spans with supported scope", the instrumentation is not active. Evaluation cannot fix missing telemetry. + +Steps 1 through 3 are framework-specific. Everything from step 4 onward, including evaluators, `EvaluationClient`, online configs, and the CLI, is identical for every framework. Compare the two `evaluate.py` files to see that they differ only in names. + +## Structure + +Each sample is self-contained and runs from its own folder: + +``` +supported-frameworks/ + openai-agents/ + openai_hr_assistant.py # agent (entrypoint) + deploy.py # deploys to AgentCore Runtime, writes agent_config.json + evaluate.py # runs on-demand + online evaluation + cleanup.py # deletes resources created by deploy.py and evaluate.py + requirements.txt # evaluation-time dependencies + README.md + llamaindex/ + llamaindex_hr_assistant.py + deploy.py + evaluate.py + cleanup.py + requirements.txt + README.md +``` + +Run each with: + +```bash +cd +uv run --frozen --with-requirements requirements.txt python deploy.py --region us-west-2 +uv run --frozen --with-requirements requirements.txt python evaluate.py --region us-west-2 +uv run --frozen --with-requirements requirements.txt python cleanup.py +``` + +See each sample's README for framework-specific setup (model access, endpoints, and ARM64 packaging notes). Each sample also shows how to re-evaluate recorded sessions from the terminal with the [AgentCore CLI](https://www.npmjs.com/package/@aws/agentcore) (`agentcore run eval --runtime-arn ... --evaluator-arn ...`). + +## Next steps + +- Explore [`../ground-truth-based-evaluation/`](../ground-truth-based-evaluation/) for the `OnDemandEvaluationDatasetRunner` and `BatchEvaluationRunner` interfaces. They work unchanged against the runtimes deployed here because the evaluation flow is framework-agnostic. +- Explore [`../custom-code-based-evaluation/`](../custom-code-based-evaluation/) for deterministic Lambda-backed evaluators. +- Add trajectory evaluators (`Builtin.TrajectoryExactOrderMatch`, `InOrderMatch`, `AnyOrderMatch`) using the `expected_trajectory` already defined in each `evaluate.py`. + +## Additional resources + +- [Supported agent frameworks](https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/supported-frameworks.html) +- [Amazon Bedrock AgentCore Developer Guide](https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/) +- [Build reliable AI agents with Amazon Bedrock AgentCore Evaluations](https://aws.amazon.com/blogs/machine-learning/build-reliable-ai-agents-with-amazon-bedrock-agentcore-evaluations/) diff --git a/01-features/06-observe-evaluate-optimize-your-agent/02-evaluate/supported-frameworks/llamaindex/.gitignore b/01-features/06-observe-evaluate-optimize-your-agent/02-evaluate/supported-frameworks/llamaindex/.gitignore new file mode 100644 index 00000000..1b41b541 --- /dev/null +++ b/01-features/06-observe-evaluate-optimize-your-agent/02-evaluate/supported-frameworks/llamaindex/.gitignore @@ -0,0 +1,3 @@ +results/ +__pycache__/ +agent_config.json diff --git a/01-features/06-observe-evaluate-optimize-your-agent/02-evaluate/supported-frameworks/llamaindex/README.md b/01-features/06-observe-evaluate-optimize-your-agent/02-evaluate/supported-frameworks/llamaindex/README.md new file mode 100644 index 00000000..a2f02b8d --- /dev/null +++ b/01-features/06-observe-evaluate-optimize-your-agent/02-evaluate/supported-frameworks/llamaindex/README.md @@ -0,0 +1,172 @@ +# Evaluate a LlamaIndex agent + +Evaluate a [LlamaIndex](https://docs.llamaindex.ai/) agent with Amazon Bedrock AgentCore Evaluations. This sample deploys the shared HR Assistant, re-implemented as a LlamaIndex `FunctionAgent` workflow, to AgentCore Runtime. It then scores the agent with built-in and custom LLM-as-a-judge evaluators in on-demand and online modes. See [LlamaIndex support in AgentCore Evaluations](https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/supported-frameworks-llamaindex.html) for the supported instrumentation libraries, scope names, span extraction rules, and agent construction best practices. + +The HR Assistant, its 5 tools, mock data, and system prompt are identical to the Strands version in [`../../utils/`](../../utils/), so ground-truth and expected responses stay consistent across the framework samples. + +## What you'll learn + +| Concept | Description | +| ----------------------------- | ----------------------------------------------------------------------------------------------------------- | +| Framework instrumentation | Make a LlamaIndex agent evaluable by adding one OpenTelemetry package without instrumentation code | +| Agent workflow structure | Build with `FunctionAgent` so the workflow/inference/tool span tree the evaluation service needs is emitted | +| AgentCore Memory | Persist multi-turn conversation history in the AgentCore Memory service, across microVM restarts | +| On-demand evaluation | Score a recorded session with built-in + custom LLM-as-a-judge evaluators via `EvaluationClient` | +| Online evaluation | Continuously score live traffic with an online evaluation config | +| CLI evaluation | Re-evaluate any session from the terminal with the AgentCore CLI | + +## Architecture + +![LlamaIndex evaluation flow across AgentCore Runtime, CloudWatch, and Evaluations](images/architecture.png) + +The PNG embeds its draw.io XML and can be opened directly in draw.io for editing. + +## How it works + +The agent is instrumented for evaluation with the OpenTelemetry LlamaIndex library (`opentelemetry-instrumentation-llamaindex`, scope `opentelemetry.instrumentation.llamaindex`). On AgentCore Runtime, AWS Distro for OpenTelemetry (ADOT) auto-discovers the library at startup, so no explicit instrumentation code is needed. The agent's spans and event records flow to CloudWatch, and AgentCore Evaluations reads them from there. + +The agent is built as a LlamaIndex agent workflow using `FunctionAgent`, following the [AgentCore best practices for LlamaIndex agents](https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/supported-frameworks-llamaindex.html): + +```python +from llama_index.core.agent.workflow import FunctionAgent +from llama_index.core.tools import FunctionTool +from bedrock_agentcore.memory import MemoryClient +from llama_index.core.base.llms.types import ChatMessage +from llama_index.llms.bedrock_converse import BedrockConverse + +tools = [FunctionTool.from_defaults(fn=get_pto_balance), ...] +agent = FunctionAgent( + tools=tools, + llm=BedrockConverse(model="us.amazon.nova-lite-v1:0", region_name=REGION), + system_prompt=SYSTEM_PROMPT, + streaming=False, +) + +# Conversation history persisted in AgentCore Memory, replayed as chat_history +memory_client = MemoryClient(region_name=REGION) +history = _load_chat_history(session_id) # [ChatMessage] from list_events +response = await agent.run(prompt, chat_history=history) +memory_client.create_event( # persist the new turn + memory_id=MEMORY_ID, actor_id=ACTOR_ID, session_id=session_id, + messages=[(prompt, "USER"), (str(response), "ASSISTANT")], +) +``` + +- Agent workflow: `FunctionAgent` emits a top-level workflow span with inference and tool child spans, which is the structure AgentCore Evaluations reconstructs a session from. +- FunctionTool: each tool is registered as a `FunctionTool` so tool spans carry recoverable names, arguments, and results. +- Text-serializable results: the tools return JSON-serializable dicts, which LlamaIndex wraps in a text block for clean capture. +- `streaming=False`: one complete inference span per model call is what the evaluation service reads. It also avoids a `BedrockConverse` streaming parser issue (`TypeError` on split tool-call input deltas). +- AgentCore Memory for conversation history: each turn is stored in the [AgentCore Memory](https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/memory.html) service via `bedrock_agentcore.memory.MemoryClient` and replayed as `chat_history`, so multi-turn context survives microVM restarts. `deploy.py` creates the memory resource and injects `AGENTCORE_MEMORY_ID`. Only USER/ASSISTANT text turns are stored. Replaying stored tool-call messages, as the official `llama-index-memory-bedrock-agentcore` integration does, trips the Bedrock Converse API's toolUse/toolResult pairing validation on the next turn. + +`FunctionAgent` is used (not `ReActAgent`) because Nova Lite supports native tool calling. If you swap in a model without tool calling, `ReActAgent` is the alternative; AgentCore then extracts the final answer from the standard `Answer:` section of its output. + +The LLM is a Bedrock model (Nova Lite via `BedrockConverse`), matching the shared Strands agent so expected responses stay identical. + +## Sample trace + +![Sanitized LlamaIndex trace showing workflow, inference, tool, and model spans](images/sample-trace.png) + +This trace was captured from the deployed sample and sanitized for publication. AgentCore Evaluations identifies `FunctionAgent.workflow` as the agent invocation from `traceloop.span.kind=workflow`. It identifies LlamaIndex inference tasks from `traceloop.span.kind=task`, and recognizes `FunctionTool.task` as a tool span because its `traceloop.entity.name` ends in `Tool.task`. The two chat spans are separate model calls before and after tool execution. The shared scope is `opentelemetry.instrumentation.llamaindex`. Repeated workflow tasks and lower-level transport spans are omitted from the figure for readability. The PNG embeds its draw.io XML and can be opened directly in draw.io for editing. + +## Prerequisites + +- Python 3.10+ +- AWS CLI configured with credentials +- Access to `us.amazon.nova-lite-v1:0` on Amazon Bedrock in your region +- Permissions for: `bedrock-agentcore:*`, `bedrock-agentcore-control:*`, `logs:*`, `iam:CreateRole`, `iam:PutRolePolicy`, `s3:PutObject`, `bedrock:InvokeModel` + +## Deploy the agent + +```bash +uv run --frozen --with-requirements requirements.txt python deploy.py --region us-west-2 +``` + +This builds an ARM64 deployment package, creates an AgentCore Memory resource (conversation history store, injected as `AGENTCORE_MEMORY_ID`), creates the AgentCore Runtime, and writes `agent_config.json` in this directory (read by `evaluate.py`). + +## Run the evaluation + +```bash +uv run --frozen --with-requirements requirements.txt python evaluate.py --region us-west-2 +``` + +The script: + +1. Creates two custom LLM-as-a-judge evaluators (`HRResponseQuality` TRACE, `HRSessionCompleteness` SESSION). +2. Invokes the deployed agent for a 3-turn session and waits ~90s for CloudWatch span ingestion. +3. Runs on-demand evaluation with `EvaluationClient` (built-in + custom evaluators, with `ReferenceInputs` ground truth). Scores are saved to `results/on_demand_results.json`. +4. Creates an online evaluation config that continuously scores live traffic with built-in evaluators. Details are saved to `results/online_eval_config.json`. + +## Expected output + +``` +[1/4] Creating custom LLM-as-a-judge evaluators ... + Creating HRResponseQuality (TRACE) ... + Creating HRSessionCompleteness (SESSION) ... + +[2/4] Invoking HR Assistant to generate a session ... + Turn 1: What is the PTO balance for employee EMP-001? + -> The PTO balance for employee EMP-001 is as follows: Total days: 15 ... + Turn 2: Please submit a PTO request for EMP-001 from 2026-07-14 to 2026-07-18. + -> The PTO request for employee EMP-001 has been submitted and approved ... + Turn 3: What is the company remote work policy? + -> Employees may work remotely up to 3 days per week ... + +[3/4] Running on-demand evaluation (EvaluationClient) ... + Evaluator Value Label + -------------------------------------------------------------------------------- + Builtin.GoalSuccessRate 1.0 Yes + Builtin.Correctness 1.0 Perfectly Correct + Builtin.Helpfulness 0.83 Very Helpful + HRResponseQuality 1.0 excellent + HRSessionCompleteness 1.0 complete + +[4/4] Creating online evaluation configuration ... + Online evaluation config created: hr_llamaindex_eval_-XXXXXXXXXX +``` + +TRACE-level evaluators (`Correctness`, `Helpfulness`, `HRResponseQuality`) return one score per turn, so the full run prints 11 results. Online evaluation results appear a few minutes later in CloudWatch at `/aws/bedrock-agentcore/evaluations/results/`, one record per evaluator per sampled turn with `gen_ai.evaluation.score.value` and `gen_ai.evaluation.explanation` attributes. + +## Evaluate from the CLI + +Once sessions exist in CloudWatch, you can re-evaluate them from the terminal with the [AgentCore CLI](https://www.npmjs.com/package/@aws/agentcore). No Python is needed. Because this sample deploys with a plain `deploy.py` (not an `agentcore` project), use the standalone flags: + +```bash +npm install -g @aws/agentcore + +AGENT_ARN=$(jq -r .agent_arn agent_config.json) +agentcore run eval \ + --runtime-arn "$AGENT_ARN" \ + --evaluator-arn Builtin.Helpfulness Builtin.Correctness \ + --region us-west-2 \ + --session-id \ + --days 1 +``` + +Ground truth can be supplied inline with `--assertion`, `--expected-trajectory`, and `--expected-response`. Omit `--session-id` to evaluate every session in the lookback window. + +## Troubleshooting ARM64 wheels + +`deploy.py` cross-compiles dependencies with `--platform manylinux2014_aarch64 --only-binary=:all:`. LlamaIndex pulls a broad dependency tree; if a transitive dependency lacks an aarch64 wheel and the install fails, either: + +- add `--no-binary=` for the offending pure-Python package, or +- build the zip on an ARM64 machine or in a `public.ecr.aws/lambda/python:3.13-arm64` container / AWS CodeBuild ARM instead of cross-compiling. + +The sample installs the full `llama-index` meta-package (not just `llama-index-core`). This is required: ADOT's auto-instrumentation checks the OpenTelemetry LlamaIndex instrumentation's declared dependency (`llama-index`) at startup and silently skips the instrumentor if only `llama-index-core` is present. The agent then emits no evaluable spans. + +## Clean up + +Run the cleanup script from this directory: + +```bash +uv run --frozen --with-requirements requirements.txt python cleanup.py +``` + +The script uses the `default` AWS profile and the region in `agent_config.json`. It deletes the online evaluation configurations and custom evaluators recorded under `results/`, then removes the AgentCore Runtime, Memory, sample-specific CloudWatch log groups, deployment package, and IAM roles. Asynchronous AgentCore deletions are checked for completion before dependent resources are removed, and the script can be run again if cleanup is interrupted. + +The shared `aws/spans` log group is retained. The regional deployment bucket is also retained when it contains objects from other samples. Use `--profile` or `--region` to override the defaults. + +## Additional resources + +- [Supported agent frameworks: LlamaIndex](https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/supported-frameworks-llamaindex.html) +- [LlamaIndex documentation](https://docs.llamaindex.ai/) +- [Amazon Bedrock AgentCore Developer Guide](https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/) diff --git a/01-features/06-observe-evaluate-optimize-your-agent/02-evaluate/supported-frameworks/llamaindex/cleanup.py b/01-features/06-observe-evaluate-optimize-your-agent/02-evaluate/supported-frameworks/llamaindex/cleanup.py new file mode 100644 index 00000000..2c5e357d --- /dev/null +++ b/01-features/06-observe-evaluate-optimize-your-agent/02-evaluate/supported-frameworks/llamaindex/cleanup.py @@ -0,0 +1,435 @@ +"""Delete AWS resources created by the LlamaIndex sample. + +The script reads agent_config.json plus optional files in results/ and removes +the sample-specific evaluation, runtime, memory, logging, S3, and IAM resources. + +Usage: + uv run --frozen --with-requirements requirements.txt python cleanup.py + [--region REGION] [--profile PROFILE] +""" + +from __future__ import annotations + +import argparse +import json +import sys +import time +from collections.abc import Callable, Sequence +from functools import partial +from pathlib import Path +from typing import Any, TypedDict + +import boto3 +from botocore.config import Config +from botocore.exceptions import BotoCoreError, ClientError + +_SCRIPT_DIR = Path(__file__).parent +_DEFAULT_CONFIG = _SCRIPT_DIR / "agent_config.json" +_DEFAULT_RESULTS_DIR = _SCRIPT_DIR / "results" +_NOT_FOUND_CODES = { + "404", + "NoSuchBucket", + "NoSuchEntity", + "NotFoundException", + "ResourceNotFoundException", +} +_Action = Callable[[], object] +_GetAction = Callable[[], dict[str, Any]] +_StatusGetter = Callable[[dict[str, Any]], str | None] + + +class EvaluationState(TypedDict): + """Resource identifiers written by evaluate.py.""" + + online_config_ids: list[str] + custom_evaluator_ids: list[str] + evaluation_role_names: list[str] + results_log_groups: list[str] + + +def _read_json(path: Path) -> dict[str, Any]: + """Read a JSON object from path. + + Args: + path: JSON file to read. + + Returns: + The decoded object, or an empty dictionary when the file is absent. + + Raises: + ValueError: If the JSON root is not an object. + """ + if not path.exists(): + return {} + value = json.loads(path.read_text()) + if not isinstance(value, dict): + raise ValueError(f"Expected a JSON object in {path}") + return value + + +def _collect_evaluation_state(results_dir: Path) -> EvaluationState: + """Merge cleanup identifiers from evaluation result files.""" + online_config_ids: set[str] = set() + evaluator_ids: set[str] = set() + evaluation_role_names: set[str] = set() + results_log_groups: set[str] = set() + + for filename in ("on_demand_results.json", "online_eval_config.json", "cleanup_state.json"): + data = _read_json(results_dir / filename) + _collect_strings(data.get("online_evaluation_config_id"), online_config_ids) + _collect_strings(data.get("online_evaluation_config_ids"), online_config_ids) + _collect_strings(data.get("config_id"), online_config_ids) + _collect_strings(data.get("custom_evaluator_ids"), evaluator_ids, exclude_builtins=True) + _collect_strings(data.get("evaluation_role_name"), evaluation_role_names) + _collect_strings(data.get("evaluation_role_names"), evaluation_role_names) + _collect_strings(data.get("results_log_group"), results_log_groups) + _collect_strings(data.get("results_log_groups"), results_log_groups) + + return { + "online_config_ids": sorted(online_config_ids), + "custom_evaluator_ids": sorted(evaluator_ids), + "evaluation_role_names": sorted(evaluation_role_names), + "results_log_groups": sorted(results_log_groups), + } + + +def _collect_strings(value: object, destination: set[str], *, exclude_builtins: bool = False) -> None: + """Collect strings from a scalar, list, or dictionary value.""" + candidates: Sequence[object] + if isinstance(value, str): + candidates = [value] + elif isinstance(value, list): + candidates = value + elif isinstance(value, dict): + candidates = list(value.values()) + else: + return + + destination.update( + candidate + for candidate in candidates + if isinstance(candidate, str) and candidate and (not exclude_builtins or not candidate.startswith("Builtin.")) + ) + + +def _role_name_from_arn(role_arn: str) -> str | None: + """Return the IAM role name from an ARN.""" + marker = ":role/" + if marker not in role_arn: + return None + return role_arn.split(marker, 1)[1].rsplit("/", 1)[-1] + + +def _error_code(error: ClientError) -> str: + """Return an AWS service error code.""" + return str(error.response.get("Error", {}).get("Code", "Unknown")) + + +def _run_step(label: str, action: _Action, failures: list[str]) -> bool: + """Run one cleanup action while treating missing resources as success.""" + print(f"Deleting {label} ...") + try: + action() + print(" [ok]") + return True + except ClientError as error: + code = _error_code(error) + if code in _NOT_FOUND_CODES: + print(" [skip] already absent") + return True + failures.append(f"{label}: {code}: {error}") + print(f" [failed] {code}: {error}") + return False + except (BotoCoreError, OSError, RuntimeError, TimeoutError) as error: + failures.append(f"{label}: {error}") + print(f" [failed] {error}") + return False + + +def _flat_status(response: dict[str, Any]) -> str | None: + """Read a top-level AgentCore resource status.""" + status = response.get("status") + return status if isinstance(status, str) else None + + +def _memory_status(response: dict[str, Any]) -> str | None: + """Read an AgentCore Memory status.""" + memory = response.get("memory") + if not isinstance(memory, dict): + return None + status = memory.get("status") + return status if isinstance(status, str) else None + + +def _delete_async_resource( + label: str, + delete_action: _Action, + get_action: _GetAction, + get_status: _StatusGetter, + failures: list[str], + *, + poll_interval: float, + timeout: float, +) -> bool: + """Delete an AgentCore resource and wait until it no longer exists.""" + print(f"Deleting {label} ...") + try: + try: + response = get_action() + except ClientError as error: + if _error_code(error) in _NOT_FOUND_CODES: + print(" [skip] already absent") + return True + raise + + if get_status(response) != "DELETING": + try: + delete_action() + except ClientError as error: + code = _error_code(error) + if code in _NOT_FOUND_CODES: + print(" [ok]") + return True + if code != "ConflictException": + raise + response = get_action() + if get_status(response) != "DELETING": + raise RuntimeError( + f"delete request conflicted; current status is {get_status(response) or 'unknown'}" + ) from error + + deadline = time.monotonic() + timeout + while True: + try: + response = get_action() + except ClientError as error: + if _error_code(error) in _NOT_FOUND_CODES: + print(" [ok]") + return True + raise + + status = get_status(response) + if time.monotonic() >= deadline: + raise TimeoutError( + f"deletion did not finish within {timeout:g} seconds; current status is {status or 'unknown'}" + ) + time.sleep(poll_interval) + except ClientError as error: + code = _error_code(error) + failures.append(f"{label}: {code}: {error}") + print(f" [failed] {code}: {error}") + return False + except (BotoCoreError, RuntimeError, TimeoutError) as error: + failures.append(f"{label}: {error}") + print(f" [failed] {error}") + return False + + +def _delete_iam_role(iam: Any, role_name: str) -> None: + """Delete all policies from an IAM role, then delete the role.""" + inline_pages = iam.get_paginator("list_role_policies").paginate(RoleName=role_name) + for page in inline_pages: + for policy_name in page.get("PolicyNames", []): + iam.delete_role_policy(RoleName=role_name, PolicyName=policy_name) + + attached_pages = iam.get_paginator("list_attached_role_policies").paginate(RoleName=role_name) + for page in attached_pages: + for policy in page.get("AttachedPolicies", []): + policy_arn = policy.get("PolicyArn") + if policy_arn: + iam.detach_role_policy(RoleName=role_name, PolicyArn=policy_arn) + + iam.delete_role(RoleName=role_name) + + +def _delete_empty_bucket(s3: Any, bucket: str) -> None: + """Delete the code bucket only when no other sample artifacts remain.""" + response = s3.list_objects_v2(Bucket=bucket, MaxKeys=1) + if response.get("KeyCount", 0): + print(" Bucket contains other objects and will be retained.") + return + s3.delete_bucket(Bucket=bucket) + + +def _require_string(config: dict[str, Any], key: str) -> str: + """Return a required non-empty string from configuration.""" + value = config.get(key) + if not isinstance(value, str) or not value: + raise ValueError(f"Missing required '{key}' in agent config") + return value + + +def _parse_args(argv: Sequence[str] | None) -> argparse.Namespace: + """Parse command-line arguments.""" + parser = argparse.ArgumentParser(description="Clean up the LlamaIndex evaluation sample") + parser.add_argument("--config", type=Path, default=_DEFAULT_CONFIG, help="Path to agent_config.json") + parser.add_argument("--results-dir", type=Path, default=_DEFAULT_RESULTS_DIR, help="Evaluation results directory") + parser.add_argument("--region", default=None, help="Override the region saved by deploy.py") + parser.add_argument("--profile", default="default", help="AWS profile (default: default)") + parser.add_argument( + "--poll-interval", type=_non_negative_float, default=5.0, help="Seconds between deletion checks" + ) + parser.add_argument( + "--timeout", + type=_non_negative_float, + default=300.0, + help="Seconds to wait for each asynchronous deletion", + ) + return parser.parse_args(argv) + + +def _non_negative_float(value: str) -> float: + """Parse a non-negative command-line number.""" + parsed = float(value) + if parsed < 0: + raise argparse.ArgumentTypeError("must be zero or greater") + return parsed + + +def main(argv: Sequence[str] | None = None) -> int: + """Run cleanup and return a process exit code.""" + args = _parse_args(argv) + try: + config = _read_json(args.config) + if not config: + raise ValueError(f"Agent config not found: {args.config}. Run deploy.py first.") + + region = args.region or _require_string(config, "region") + agent_id = _require_string(config, "agent_id") + evaluation = _collect_evaluation_state(args.results_dir) + + session = boto3.Session(profile_name=args.profile, region_name=region) + client_config = Config(retries={"mode": "adaptive", "total_max_attempts": 5}) + identity = session.client("sts", config=client_config).get_caller_identity() + control = session.client("bedrock-agentcore-control", config=client_config) + logs = session.client("logs", config=client_config) + iam = session.client("iam", config=client_config) + s3 = session.client("s3", config=client_config) + + print(f"AWS account: {identity['Account']}") + print(f"Region: {region}") + print(f"Runtime: {agent_id}") + print() + + failures: list[str] = [] + + online_configs_deleted = True + for online_config_id in evaluation["online_config_ids"]: + deleted = _delete_async_resource( + f"online evaluation config {online_config_id}", + lambda: control.delete_online_evaluation_config( + onlineEvaluationConfigId=online_config_id, + ), + lambda: control.get_online_evaluation_config( + onlineEvaluationConfigId=online_config_id, + ), + _flat_status, + failures, + poll_interval=args.poll_interval, + timeout=args.timeout, + ) + online_configs_deleted = deleted and online_configs_deleted + + if online_configs_deleted: + for evaluator_id in evaluation["custom_evaluator_ids"]: + _delete_async_resource( + f"custom evaluator {evaluator_id}", + partial(control.delete_evaluator, evaluatorId=evaluator_id), + partial(control.get_evaluator, evaluatorId=evaluator_id), + _flat_status, + failures, + poll_interval=args.poll_interval, + timeout=args.timeout, + ) + else: + print("Skipping evaluators and dependent resources because an online evaluation config remains.") + + runtime_deleted = False + if online_configs_deleted: + runtime_deleted = _delete_async_resource( + f"AgentCore Runtime {agent_id}", + lambda: control.delete_agent_runtime(agentRuntimeId=agent_id), + lambda: control.get_agent_runtime(agentRuntimeId=agent_id), + _flat_status, + failures, + poll_interval=args.poll_interval, + timeout=args.timeout, + ) + else: + print("Skipping AgentCore Runtime because an online evaluation config remains.") + + memory_id = config.get("memory_id") + if runtime_deleted and isinstance(memory_id, str) and memory_id: + _delete_async_resource( + f"AgentCore Memory {memory_id}", + lambda: control.delete_memory(memoryId=memory_id), + lambda: control.get_memory(memoryId=memory_id), + _memory_status, + failures, + poll_interval=args.poll_interval, + timeout=args.timeout, + ) + + runtime_log_group = config.get("cw_log_group") + if runtime_deleted and isinstance(runtime_log_group, str) and runtime_log_group: + _run_step( + f"runtime log group {runtime_log_group}", + lambda: logs.delete_log_group(logGroupName=runtime_log_group), + failures, + ) + + if online_configs_deleted: + for results_log_group in evaluation["results_log_groups"]: + _run_step( + f"evaluation results log group {results_log_group}", + partial(logs.delete_log_group, logGroupName=results_log_group), + failures, + ) + + s3_bucket = config.get("s3_bucket") + s3_key = config.get("s3_key") + if runtime_deleted and isinstance(s3_bucket, str) and s3_bucket and isinstance(s3_key, str) and s3_key: + _run_step( + f"S3 object s3://{s3_bucket}/{s3_key}", + lambda: s3.delete_object(Bucket=s3_bucket, Key=s3_key), + failures, + ) + _run_step( + f"empty S3 bucket {s3_bucket}", + lambda: _delete_empty_bucket(s3, s3_bucket), + failures, + ) + + if online_configs_deleted: + for evaluation_role_name in evaluation["evaluation_role_names"]: + _run_step( + f"evaluation IAM role {evaluation_role_name}", + partial(_delete_iam_role, iam, evaluation_role_name), + failures, + ) + + role_arn = config.get("role_arn") + runtime_role_name = _role_name_from_arn(role_arn) if isinstance(role_arn, str) else None + if runtime_deleted and runtime_role_name: + _run_step( + f"runtime IAM role {runtime_role_name}", + lambda: _delete_iam_role(iam, runtime_role_name), + failures, + ) + + if failures: + print("\nCleanup finished with failures:") + for failure in failures: + print(f" - {failure}") + print("Re-run cleanup.py after resolving the reported errors.") + return 1 + + print("\nCleanup complete.") + return 0 + except (BotoCoreError, ClientError, OSError, ValueError, json.JSONDecodeError) as error: + print(f"Cleanup failed: {error}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/01-features/06-observe-evaluate-optimize-your-agent/02-evaluate/supported-frameworks/llamaindex/deploy.py b/01-features/06-observe-evaluate-optimize-your-agent/02-evaluate/supported-frameworks/llamaindex/deploy.py new file mode 100644 index 00000000..2a13ba87 --- /dev/null +++ b/01-features/06-observe-evaluate-optimize-your-agent/02-evaluate/supported-frameworks/llamaindex/deploy.py @@ -0,0 +1,297 @@ +"""Deploy the LlamaIndex HR Assistant to AgentCore Runtime using the bedrock-agentcore SDK. + +Packages the agent source and its dependencies into a zip, uploads to S3, creates an +AgentCore Runtime, and polls until READY. Saves connection details to agent_config.json +in this directory for use by evaluate.py. + +Usage: + python deploy.py [--region REGION] + +Output: + agent_config.json — AGENT_ID, AGENT_ARN, CW_LOG_GROUP, REGION + +Deployment steps: + 1. Create an IAM execution role for the runtime + 2. Create an AgentCore Memory resource (conversation history store) + 3. Package llamaindex_hr_assistant.py + ARM64 dependencies into a zip + 4. Upload the zip to S3 + 5. Create an AgentCore Runtime via create_agent_runtime (codeConfiguration), + injecting AGENTCORE_MEMORY_ID as an environment variable + 6. Poll until READY + 7. Write agent_config.json + +The runtime uses a Bedrock model (Nova Lite) via BedrockConverse, so the +bedrock:InvokeModel* permission granted below is sufficient. + +See https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/getting-started-custom.html +""" + +import argparse +import json +import os +import shutil +import subprocess +import sys +import time +import uuid +import zipfile +from pathlib import Path + +import boto3 +from boto3.session import Session + +# --------------------------------------------------------------------------- +# Config +# --------------------------------------------------------------------------- + +_SCRIPT_DIR = Path(__file__).parent +_CONFIG_FILE = _SCRIPT_DIR / "agent_config.json" +_AGENT_FILE = "llamaindex_hr_assistant.py" + +# Bundled into the deployment zip (ARM64). The full llama-index meta-package is +# required (not just llama-index-core): ADOT's auto-instrumentation checks the +# instrumentation package's declared "instruments" dependency (llama-index) and +# skips the instrumentor if it is missing. llama-index-llms-bedrock-converse +# provides the LLM. +_PACKAGES = [ + "llama-index", + "llama-index-llms-bedrock-converse", + "opentelemetry-instrumentation-llamaindex>=0.61.0", + "bedrock-agentcore", + "aws-opentelemetry-distro", +] + +parser = argparse.ArgumentParser(description="Deploy the LlamaIndex HR Assistant to AgentCore Runtime") +parser.add_argument("--region", default=None, help="AWS region (default: boto3 session region)") +args = parser.parse_args() + +REGION = args.region or Session().region_name or "us-west-2" +print(f"Region: {REGION}") + +_sts = boto3.client("sts", region_name=REGION) +_ACCOUNT_ID = _sts.get_caller_identity()["Account"] +_iam = boto3.client("iam", region_name=REGION) +_s3 = boto3.client("s3", region_name=REGION) +_ctrl = boto3.client("bedrock-agentcore-control", region_name=REGION) + +_AGENT_NAME = f"hr_llamaindex_{uuid.uuid4().hex[:8]}" +_ROLE_NAME = f"{_AGENT_NAME}_role" +_S3_BUCKET = f"bedrock-agentcore-code-{_ACCOUNT_ID}-{REGION}" +_S3_KEY = f"{_AGENT_NAME}/deployment_package.zip" +_BUILD_DIR = Path(f"/tmp/{_AGENT_NAME}_build") # nosec B108 + +# --------------------------------------------------------------------------- +# 1. IAM execution role +# --------------------------------------------------------------------------- + +_TRUST = json.dumps( + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": {"Service": "bedrock-agentcore.amazonaws.com"}, + "Action": "sts:AssumeRole", + "Condition": { + "StringEquals": {"aws:SourceAccount": _ACCOUNT_ID}, + "ArnLike": {"aws:SourceArn": f"arn:aws:bedrock-agentcore:*:{_ACCOUNT_ID}:runtime/*"}, + }, + } + ], + } +) + +_POLICY = json.dumps( + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "bedrock:InvokeModel", + "bedrock:InvokeModelWithResponseStream", + "bedrock-agentcore:CreateEvent", + "bedrock-agentcore:ListEvents", + "bedrock-agentcore:GetEvent", + "bedrock-agentcore:GetMemory", + "bedrock-agentcore:ListMemoryRecords", + "bedrock-agentcore:RetrieveMemoryRecords", + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:PutLogEvents", + "logs:DescribeLogGroups", + "logs:DescribeLogStreams", + "xray:PutTraceSegments", + "xray:PutTelemetryRecords", + "xray:GetSamplingRules", + "xray:GetSamplingTargets", + "cloudwatch:PutMetricData", + ], + "Resource": "*", + } + ], + } +) + +print(f"\n[1/5] Creating IAM role '{_ROLE_NAME}' ...") +try: + _ROLE_ARN = _iam.create_role(RoleName=_ROLE_NAME, AssumeRolePolicyDocument=_TRUST)["Role"]["Arn"] + print(f" Created: {_ROLE_ARN}") +except _iam.exceptions.EntityAlreadyExistsException: + _ROLE_ARN = _iam.get_role(RoleName=_ROLE_NAME)["Role"]["Arn"] + print(f" Already exists: {_ROLE_ARN}") + +_iam.put_role_policy( + RoleName=_ROLE_NAME, + PolicyName=f"{_AGENT_NAME}_policy", + PolicyDocument=_POLICY, +) +print(" Policy attached. Waiting 10s for IAM propagation ...") +time.sleep(10) + +# --------------------------------------------------------------------------- +# 2. Create AgentCore Memory (conversation history store) +# --------------------------------------------------------------------------- + +print(f"\n[2/6] Creating AgentCore Memory '{_AGENT_NAME}_memory' ...") +_mem_resp = _ctrl.create_memory( + name=f"{_AGENT_NAME}_memory", + description="Short-term conversation memory for the HR Assistant sample", + eventExpiryDuration=7, +) +MEMORY_ID = _mem_resp["memory"]["id"] +print(f" Memory ID: {MEMORY_ID}") +for _elapsed in range(0, 300, 10): + _mstatus = _ctrl.get_memory(memoryId=MEMORY_ID)["memory"]["status"] + if _mstatus == "ACTIVE": + break + if _mstatus == "FAILED": + raise RuntimeError("Memory creation failed") + time.sleep(10) +print(f" Memory status: {_mstatus}") + +# --------------------------------------------------------------------------- +# 3. Build deployment package (ARM64) +# --------------------------------------------------------------------------- + +print("\n[3/6] Building deployment package ...") +if _BUILD_DIR.exists(): + shutil.rmtree(_BUILD_DIR) +_PKG = _BUILD_DIR / "pkg" +_PKG.mkdir(parents=True) + +subprocess.run( + [ + sys.executable, + "-m", + "pip", + "install", + *_PACKAGES, + "-t", + str(_PKG), + "--platform", + "manylinux2014_aarch64", + "--only-binary=:all:", + "--python-version", + "3.13", + # This is an isolated --target install; suppress pip's dependency check + # against the ambient environment (false positives) and version notice. + "--no-warn-conflicts", + "--disable-pip-version-check", + "--quiet", + ], + check=True, +) +shutil.copy(_SCRIPT_DIR / _AGENT_FILE, _PKG / _AGENT_FILE) + +_ZIP = _BUILD_DIR / "deployment_package.zip" +with zipfile.ZipFile(_ZIP, "w", zipfile.ZIP_DEFLATED) as zf: + for root, _, files in os.walk(_PKG): + for f in files: + if f.endswith(".pyc") or "__pycache__" in root: + continue + full = Path(root) / f + zf.write(full, full.relative_to(_PKG)) +print(f" Package: {_ZIP} ({_ZIP.stat().st_size / 1024 / 1024:.1f} MB)") + +# --------------------------------------------------------------------------- +# 3. Upload to S3 +# --------------------------------------------------------------------------- + +print("\n[4/6] Uploading to S3 ...") +try: + if REGION == "us-east-1": + _s3.create_bucket(Bucket=_S3_BUCKET) + else: + _s3.create_bucket( + Bucket=_S3_BUCKET, + CreateBucketConfiguration={"LocationConstraint": REGION}, + ) + print(f" Created bucket: {_S3_BUCKET}") +except Exception: + print(f" Bucket exists: {_S3_BUCKET}") +_s3.upload_file(str(_ZIP), _S3_BUCKET, _S3_KEY) +print(f" Uploaded: s3://{_S3_BUCKET}/{_S3_KEY}") + +# --------------------------------------------------------------------------- +# 4. Create AgentCore Runtime +# --------------------------------------------------------------------------- + +print(f"\n[5/6] Creating AgentCore Runtime '{_AGENT_NAME}' ...") +_resp = _ctrl.create_agent_runtime( + agentRuntimeName=_AGENT_NAME, + agentRuntimeArtifact={ + "codeConfiguration": { + "code": {"s3": {"bucket": _S3_BUCKET, "prefix": _S3_KEY}}, + "runtime": "PYTHON_3_13", + "entryPoint": ["opentelemetry-instrument", _AGENT_FILE], + } + }, + networkConfiguration={"networkMode": "PUBLIC"}, + roleArn=_ROLE_ARN, + environmentVariables={"AGENTCORE_MEMORY_ID": MEMORY_ID}, +) +AGENT_ID = _resp["agentRuntimeId"] +print(f" Runtime ID: {AGENT_ID}") + +# --------------------------------------------------------------------------- +# 5. Poll until READY +# --------------------------------------------------------------------------- + +print("\n[6/6] Waiting for READY ...") +for _elapsed in range(0, 600, 15): + _status = _ctrl.get_agent_runtime(agentRuntimeId=AGENT_ID).get("status", "UNKNOWN") + print(f" [{_elapsed:>3}s] {_status}") + if _status in ("READY", "ACTIVE"): + break + if "FAILED" in _status: + raise RuntimeError(f"Deploy failed: {_status}") + time.sleep(15) +else: + raise TimeoutError("Agent did not reach READY in 600s") + +AGENT_ARN = _ctrl.get_agent_runtime(agentRuntimeId=AGENT_ID)["agentRuntimeArn"] +CW_LOG_GROUP = f"/aws/bedrock-agentcore/runtimes/{AGENT_ID}-DEFAULT" + +# --------------------------------------------------------------------------- +# 6. Save agent_config.json +# --------------------------------------------------------------------------- + +_config = { + "agent_id": AGENT_ID, + "agent_arn": AGENT_ARN, + "cw_log_group": CW_LOG_GROUP, + "region": REGION, + "role_arn": _ROLE_ARN, + "s3_bucket": _S3_BUCKET, + "s3_key": _S3_KEY, + "memory_id": MEMORY_ID, +} +_CONFIG_FILE.write_text(json.dumps(_config, indent=2)) + +print("\nDeploy complete.") +print(f" AGENT_ID : {AGENT_ID}") +print(f" AGENT_ARN : {AGENT_ARN}") +print(f" CW_LOG_GROUP : {CW_LOG_GROUP}") +print(f" MEMORY_ID : {MEMORY_ID}") +print(f" Config saved : {_CONFIG_FILE}") diff --git a/01-features/06-observe-evaluate-optimize-your-agent/02-evaluate/supported-frameworks/llamaindex/evaluate.py b/01-features/06-observe-evaluate-optimize-your-agent/02-evaluate/supported-frameworks/llamaindex/evaluate.py new file mode 100644 index 00000000..6f7fced9 --- /dev/null +++ b/01-features/06-observe-evaluate-optimize-your-agent/02-evaluate/supported-frameworks/llamaindex/evaluate.py @@ -0,0 +1,598 @@ +""" +Evaluate the LlamaIndex HR Assistant with AgentCore Evaluations. + +This is the same evaluation flow used by the framework-agnostic samples in +../../ — the driver only invokes the deployed runtime and calls the evaluation +APIs, so it is identical regardless of the agent framework. The agent behind +agent_config.json here is a LlamaIndex FunctionAgent workflow (see +llamaindex_hr_assistant.py) instrumented with the OpenTelemetry LlamaIndex +library (scope opentelemetry.instrumentation.llamaindex). + +Two evaluation modes are demonstrated: + + 1. On-Demand Evaluation (EvaluationClient) + Invoke the agent for a session, then evaluate the recorded CloudWatch + spans immediately. Built-in evaluators + custom LLM-as-a-judge evaluators + run in the same call. Use for spot-checks and CI/CD regression tests. + + 2. Online Evaluation (create_online_evaluation_config) + Create a persistent config that continuously monitors the agent's live + traffic and scores every sampled session automatically. + +Usage: + python evaluate.py [--region REGION] [--config PATH] + +Args: + --region AWS region (default: from agent_config.json or boto3 session) + --config Path to agent_config.json written by deploy.py + (default: ./agent_config.json) + +Prerequisites: + 1. Deploy the LlamaIndex HR Assistant: + python deploy.py [--region REGION] + 2. Install evaluation dependencies: + pip install -r requirements.txt + +Outputs: + results/on_demand_results.json - EvaluationClient scores + results/online_eval_config.json - Online evaluation config details +""" + +import argparse +import json +import sys +import time +import uuid +from pathlib import Path + +import boto3 +from boto3.session import Session + +# ============================================================ +# 0. Parse args and load agent config +# ============================================================ + +_SCRIPT_DIR = Path(__file__).parent +_DEFAULT_CONFIG = _SCRIPT_DIR / "agent_config.json" +_RESULTS_DIR = _SCRIPT_DIR / "results" +_RESULTS_DIR.mkdir(exist_ok=True) +_CLEANUP_STATE_PATH = _RESULTS_DIR / "cleanup_state.json" + + +def _load_cleanup_state() -> dict[str, object]: + """Load identifiers saved by previous complete or partial runs.""" + if not _CLEANUP_STATE_PATH.exists(): + return {} + state = json.loads(_CLEANUP_STATE_PATH.read_text()) + if not isinstance(state, dict): + raise ValueError(f"Expected a JSON object in {_CLEANUP_STATE_PATH}") + return state + + +_cleanup_state = _load_cleanup_state() + + +def _save_cleanup_state() -> None: + """Persist resource identifiers for cleanup after partial or complete runs.""" + _CLEANUP_STATE_PATH.write_text(json.dumps(_cleanup_state, indent=2)) + + +def _remember_cleanup_value(key: str, value: str) -> None: + """Append a resource identifier to cleanup state without duplicates.""" + existing = _cleanup_state.get(key) + values = [item for item in existing if isinstance(item, str)] if isinstance(existing, list) else [] + if value not in values: + values.append(value) + _cleanup_state[key] = values + _save_cleanup_state() + + +parser = argparse.ArgumentParser(description="Evaluate the LlamaIndex HR Assistant") +parser.add_argument("--region", default=None, help="AWS region") +parser.add_argument( + "--config", + default=str(_DEFAULT_CONFIG), + help="Path to agent_config.json (written by deploy.py)", +) +args = parser.parse_args() + +_config_path = Path(args.config) +if not _config_path.exists(): + print(f"ERROR: Agent config not found at {_config_path}") + print("Run deploy.py first: python deploy.py") + sys.exit(1) + +_cfg = json.loads(_config_path.read_text()) +AGENT_ID = _cfg["agent_id"] +AGENT_ARN = _cfg["agent_arn"] +CW_LOG_GROUP = _cfg["cw_log_group"] +REGION = args.region or _cfg.get("region") or Session().region_name or "us-west-2" + +ACCOUNT_ID = boto3.client("sts", region_name=REGION).get_caller_identity()["Account"] + +# Derive OTel service name from agent ARN: +# ARN format: arn:aws:bedrock-agentcore:{region}:{account}:runtime/{id} +_runtime_id = AGENT_ARN.split("/")[-1] +_agent_runtime_name = _runtime_id.rsplit("-", 1)[0] +OTEL_SERVICE_NAME = f"{_agent_runtime_name}.DEFAULT" + +print("=" * 60) +print("LlamaIndex HR Assistant — AgentCore Evaluation") +print("=" * 60) +print(f" Region : {REGION}") +print(f" Agent ID : {AGENT_ID}") +print(f" Agent ARN : {AGENT_ARN}") +print(f" CW Log Group : {CW_LOG_GROUP}") +print(f" OTel Service : {OTEL_SERVICE_NAME}") + +agentcore_client = boto3.client("bedrock-agentcore", region_name=REGION) +_cp = boto3.client("bedrock-agentcore-control", region_name=REGION) +iam_client = boto3.client("iam") + +# ============================================================ +# 1. Create custom LLM-as-a-judge evaluators +# ============================================================ +# +# Custom evaluators define quality criteria in natural language. +# The service substitutes ground-truth placeholders at evaluation time. +# +# Two evaluator types useful for HR assistants: +# - TRACE-level: score each agent response against the expected answer +# - SESSION-level: check whether the right tools were called and all +# assertions are satisfied across the whole conversation + +print("\n[1/4] Creating custom LLM-as-a-judge evaluators ...") + +_SUFFIX = uuid.uuid4().hex[:8] + +# ---- Trace-level: HR response quality -------------------------------- +print(" Creating HRResponseQuality (TRACE) ...") +_resp_quality = _cp.create_evaluator( + evaluatorName=f"HRResponseQuality_llamaindex_{_SUFFIX}", + level="TRACE", + evaluatorConfig={ + "llmAsAJudge": { + "instructions": ( + "You are evaluating an HR assistant chatbot response.\n\n" + "Agent response: {assistant_turn}\n\n" + "Rate the quality of the agent's response on the following criteria:\n" + "1. ACCURACY: Facts, numbers, and dates are stated confidently and consistently\n" + "2. COMPLETENESS: The response fully addresses the user's request\n" + "3. PROFESSIONALISM: Tone is appropriate for an HR context\n\n" + "Assign a single overall quality rating." + ), + "ratingScale": { + "numerical": [ + { + "value": 0.0, + "label": "poor", + "definition": "Response is inaccurate, incomplete, or unprofessional.", + }, + { + "value": 0.5, + "label": "acceptable", + "definition": "Response is mostly correct but missing details or slightly off.", + }, + { + "value": 1.0, + "label": "excellent", + "definition": "Response is accurate, complete, and professionally written.", + }, + ] + }, + "modelConfig": { + "bedrockEvaluatorModelConfig": { + "modelId": "us.amazon.nova-pro-v1:0", + "inferenceConfig": {"maxTokens": 1024}, + } + }, + } + }, +) +CUSTOM_RESPONSE_QUALITY_ID = _resp_quality["evaluatorId"] +_remember_cleanup_value("custom_evaluator_ids", CUSTOM_RESPONSE_QUALITY_ID) +print(f" evaluatorId: {CUSTOM_RESPONSE_QUALITY_ID}") + +# ---- Session-level: HR session completeness -------------------------- +print(" Creating HRSessionCompleteness (SESSION) ...") +_session_check = _cp.create_evaluator( + evaluatorName=f"HRSessionCompleteness_llamaindex_{_SUFFIX}", + level="SESSION", + evaluatorConfig={ + "llmAsAJudge": { + "instructions": ( + "You are reviewing a complete HR assistant conversation.\n\n" + "Expected tool trajectory: {expected_tool_trajectory}\n" + "Actual tool trajectory: {actual_tool_trajectory}\n" + "Session assertions: {assertions}\n\n" + "Evaluate whether the agent:\n" + "1. Called the expected tools (in any order)\n" + "2. Satisfied all session assertions\n" + "3. Reached a successful resolution for the user's request\n\n" + "Rate the overall session completeness." + ), + "ratingScale": { + "numerical": [ + { + "value": 0.0, + "label": "incomplete", + "definition": "Agent failed to call required tools or left the request unresolved.", + }, + { + "value": 0.5, + "label": "partial", + "definition": "Agent partially fulfilled the request — some tools missing or assertions unmet.", + }, + { + "value": 1.0, + "label": "complete", + "definition": "Agent called all expected tools and satisfied every assertion.", + }, + ] + }, + "modelConfig": { + "bedrockEvaluatorModelConfig": { + "modelId": "us.amazon.nova-pro-v1:0", + "inferenceConfig": {"maxTokens": 1024}, + } + }, + } + }, +) +CUSTOM_SESSION_COMPLETENESS_ID = _session_check["evaluatorId"] +_remember_cleanup_value("custom_evaluator_ids", CUSTOM_SESSION_COMPLETENESS_ID) +print(f" evaluatorId: {CUSTOM_SESSION_COMPLETENESS_ID}") + +# ============================================================ +# 2. Invoke agent to generate a session +# ============================================================ +# +# The LlamaIndex HR assistant is already deployed (deploy.py). +# We invoke it for a multi-turn session so there are CloudWatch spans +# to evaluate. A unique runtimeSessionId groups all turns together. + +print("\n[2/4] Invoking HR Assistant to generate a session ...") + +SESSION_ID = f"llamaindex-hr-eval-{uuid.uuid4()}" +print(f" Session ID: {SESSION_ID}") + +TURNS = [ + "What is the PTO balance for employee EMP-001?", + "Please submit a PTO request for EMP-001 from 2026-07-14 to 2026-07-18.", + "What is the company remote work policy?", +] + +EXPECTED_RESPONSES = [ + "Employee EMP-001 has 10 remaining PTO days out of 15 total (5 days used).", + "PTO request submitted for EMP-001 from 2026-07-14 to 2026-07-18. Request ID: PTO-2026-NNN.", + "The company allows up to 3 days of remote work per week. Core hours are 10am–3pm.", +] + +EXPECTED_TRAJECTORY = ["get_pto_balance", "submit_pto_request", "lookup_hr_policy"] + +ASSERTIONS = [ + "Agent called get_pto_balance with employee_id=EMP-001", + "Agent reported 10 remaining PTO days", + "Agent submitted a PTO request and returned a request ID", + "Agent described the remote work policy", +] + + +def _invoke_turn(prompt: str) -> str: + resp = agentcore_client.invoke_agent_runtime( + agentRuntimeArn=AGENT_ARN, + qualifier="DEFAULT", + runtimeSessionId=SESSION_ID, + payload=json.dumps({"prompt": prompt}).encode("utf-8"), + ) + raw = resp["response"].read().decode("utf-8") + parts = [] + for line in raw.splitlines(): + if line.startswith("data: "): + chunk = line[len("data: ") :] + try: + chunk = json.loads(chunk) + except Exception: + pass + parts.append(str(chunk)) + return "".join(parts) if parts else raw + + +for i, (prompt, expected) in enumerate(zip(TURNS, EXPECTED_RESPONSES), 1): + print(f" Turn {i}: {prompt[:70]}") + reply = _invoke_turn(prompt) + print(f" -> {reply[:100]}") + +print("\n Waiting 90s for CloudWatch span ingestion ...") +time.sleep(90) +print(" Ready for evaluation.") + +# ============================================================ +# 3. On-Demand Evaluation with EvaluationClient +# ============================================================ +# +# EvaluationClient evaluates the recorded session spans from CloudWatch. +# You can mix built-in evaluators with your custom LLM-as-a-judge evaluators +# in the same call. Provide ReferenceInputs ground truth to unlock evaluators +# that require expected responses or trajectories. + +from bedrock_agentcore.evaluation import EvaluationClient # noqa: E402 +from bedrock_agentcore.evaluation.client import ReferenceInputs # noqa: E402 +from datetime import timedelta # noqa: E402 + +print("\n[3/4] Running on-demand evaluation (EvaluationClient) ...") + +ec = EvaluationClient(region_name=REGION) + +# Pre-populate the evaluator level cache — required for Builtin.* evaluators +# because the SDK cannot resolve their level via GetEvaluator API. +ec._evaluator_level_cache.update( + { + "Builtin.GoalSuccessRate": "SESSION", + "Builtin.Correctness": "TRACE", + "Builtin.Helpfulness": "TRACE", + CUSTOM_RESPONSE_QUALITY_ID: "TRACE", + CUSTOM_SESSION_COMPLETENESS_ID: "SESSION", + } +) + +EVALUATOR_IDS = [ + "Builtin.GoalSuccessRate", # SESSION: did the agent meet the user's goal? + "Builtin.Correctness", # TRACE: is each response factually correct? + "Builtin.Helpfulness", # TRACE: was each response helpful? + CUSTOM_RESPONSE_QUALITY_ID, # TRACE: HR-specific response quality + CUSTOM_SESSION_COMPLETENESS_ID, # SESSION: did all assertions pass? +] + +# ReferenceInputs provide ground truth for evaluators that need it. +REFERENCE_INPUTS = ReferenceInputs( + assertions=ASSERTIONS, + expected_trajectory=EXPECTED_TRAJECTORY, + expected_response=EXPECTED_RESPONSES[-1], +) + +on_demand_results = ec.run( + evaluator_ids=EVALUATOR_IDS, + agent_id=AGENT_ID, + session_id=SESSION_ID, + look_back_time=timedelta(hours=1), + reference_inputs=REFERENCE_INPUTS, +) + +# Display results +print(f"\n Received {len(on_demand_results)} result(s):\n") +print(f" {'Evaluator':<45} {'Value':<8} {'Label'}") +print(" " + "-" * 80) + +for result in on_demand_results: + evaluator_id = result.get("evaluatorId", "") + name = ( + evaluator_id + if evaluator_id.startswith("Builtin.") + else ("HRResponseQuality" if evaluator_id == CUSTOM_RESPONSE_QUALITY_ID else "HRSessionCompleteness") + ) + value = result.get("value", result.get("score", "N/A")) + label = result.get("label", result.get("rating", "N/A")) + error = result.get("errorCode") + if error: + label = f"ERR:{error}" + print(f" {name:<45} {str(value):<8} {str(label)}") + +# Save results +_results_path = _RESULTS_DIR / "on_demand_results.json" +_results_path.write_text( + json.dumps( + { + "session_id": SESSION_ID, + "evaluators": EVALUATOR_IDS, + "custom_evaluator_ids": { + "HRResponseQuality": CUSTOM_RESPONSE_QUALITY_ID, + "HRSessionCompleteness": CUSTOM_SESSION_COMPLETENESS_ID, + }, + "results": on_demand_results, + }, + indent=2, + default=str, + ) +) +print(f"\n Results saved: {_results_path}") + +# ============================================================ +# 4. Online Evaluation Configuration +# ============================================================ +# +# Online evaluation monitors live agent traffic continuously. +# Create a config once; it evaluates every sampled session automatically. +# +# Note: Once a config is ENABLED, its evaluators are LOCKED. +# To update an evaluator: disable the config → update → re-enable. + +print("\n[4/4] Creating online evaluation configuration ...") + +# ---- 4a. IAM role for the evaluation service ------------------------- +ONLINE_EVAL_ROLE_NAME = f"AgentCoreOnlineEvalLlamaIndex_{_SUFFIX}" +ONLINE_EVAL_ROLE_ARN = f"arn:aws:iam::{ACCOUNT_ID}:role/{ONLINE_EVAL_ROLE_NAME}" + +_trust_policy = json.dumps( + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": {"Service": "bedrock-agentcore.amazonaws.com"}, + "Action": "sts:AssumeRole", + } + ], + } +) + +_inline_policy = json.dumps( + { + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "CloudWatchLogsReadWrite", + "Effect": "Allow", + "Action": [ + "logs:FilterLogEvents", + "logs:GetLogEvents", + "logs:DescribeLogGroups", + "logs:DescribeLogStreams", + "logs:StartQuery", + "logs:GetQueryResults", + "logs:StopQuery", + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:PutLogEvents", + ], + "Resource": "*", + }, + { + "Sid": "BedrockInvokeForJudge", + "Effect": "Allow", + "Action": [ + "bedrock:InvokeModel", + "bedrock:InvokeModelWithResponseStream", + ], + "Resource": "*", + }, + ], + } +) + +try: + iam_client.get_role(RoleName=ONLINE_EVAL_ROLE_NAME) + print(f" Using existing IAM role: {ONLINE_EVAL_ROLE_ARN}") +except iam_client.exceptions.NoSuchEntityException: + iam_client.create_role( + RoleName=ONLINE_EVAL_ROLE_NAME, + AssumeRolePolicyDocument=_trust_policy, + Description="Execution role for AgentCore online LLM-as-a-judge evaluation", + ) + print(f" Created IAM role: {ONLINE_EVAL_ROLE_ARN}") + +_remember_cleanup_value("evaluation_role_names", ONLINE_EVAL_ROLE_NAME) +iam_client.put_role_policy( + RoleName=ONLINE_EVAL_ROLE_NAME, + PolicyName="AgentCoreOnlineEvalPolicy", + PolicyDocument=_inline_policy, +) +print(" Waiting 10s for IAM propagation ...") +time.sleep(10) + +# ---- 4b. Create online evaluation config ---------------------------- +# Config name: alphanumeric + underscores only (no hyphens) +ONLINE_EVAL_CONFIG_NAME = f"hr_llamaindex_eval_{_SUFFIX}" + +# Note: Custom evaluators that use reference input placeholders +# ({expected_response}, {assertions}, etc.) require ground truth and therefore +# can only be used in on-demand evaluation. Online evaluation evaluates live +# traffic where no ground truth is available, so only built-in evaluators +# (or custom evaluators without reference inputs) are supported here. +_ONLINE_EVALUATORS = [ + "Builtin.GoalSuccessRate", + "Builtin.Correctness", + "Builtin.Helpfulness", +] + +print(f" Config name : {ONLINE_EVAL_CONFIG_NAME}") +print(f" Log group : {CW_LOG_GROUP}") +print(f" OTel service : {OTEL_SERVICE_NAME}") +print(f" Evaluators : {', '.join(_ONLINE_EVALUATORS)}") +print(" Note: Custom evaluators with reference inputs are on-demand only") + +_online_resp = _cp.create_online_evaluation_config( + onlineEvaluationConfigName=ONLINE_EVAL_CONFIG_NAME, + # 100% sampling in this example; lower for high-traffic production agents + rule={"samplingConfig": {"samplingPercentage": 100.0}}, + dataSourceConfig={ + "cloudWatchLogs": { + "logGroupNames": [CW_LOG_GROUP], + "serviceNames": [OTEL_SERVICE_NAME], + } + }, + evaluators=[{"evaluatorId": eid} for eid in _ONLINE_EVALUATORS], + evaluationExecutionRoleArn=ONLINE_EVAL_ROLE_ARN, + enableOnCreate=True, +) + +ONLINE_CONFIG_ID = _online_resp["onlineEvaluationConfigId"] +ONLINE_CONFIG_ARN = _online_resp.get("onlineEvaluationConfigArn", "") +_remember_cleanup_value("online_evaluation_config_ids", ONLINE_CONFIG_ID) +_remember_cleanup_value( + "results_log_groups", + f"/aws/bedrock-agentcore/evaluations/results/{ONLINE_CONFIG_ID}", +) + +print("\n Online evaluation config created:") +print(f" ID : {ONLINE_CONFIG_ID}") +print(f" ARN : {ONLINE_CONFIG_ARN}") +print() +print(" The config is now ACTIVE. Every new HR assistant session will be") +print(" automatically evaluated with built-in evaluators.") +print(" Results appear in CloudWatch at:") +print(f" /aws/bedrock-agentcore/evaluations/results/{ONLINE_CONFIG_ID}") + +# ---- 4c. Invoke agent to trigger online evaluation ------------------ +print("\n Invoking agent to trigger a live online evaluation ...") + +_online_session = f"online-llamaindex-{uuid.uuid4()}" +_online_prompts = [ + "What is the PTO balance for employee EMP-042?", + "What health insurance options does the company offer?", +] + +for prompt in _online_prompts: + print(f" > {prompt[:70]}") + reply = agentcore_client.invoke_agent_runtime( + agentRuntimeArn=AGENT_ARN, + qualifier="DEFAULT", + runtimeSessionId=_online_session, + payload=json.dumps({"prompt": prompt}).encode("utf-8"), + ) + reply.get("response", b"").read() # consume stream + +print(" Online evaluation will score this session automatically.") +print(" Results appear in CloudWatch within a few minutes.") + +# Save online eval config details +_online_path = _RESULTS_DIR / "online_eval_config.json" +_online_path.write_text( + json.dumps( + { + "config_name": ONLINE_EVAL_CONFIG_NAME, + "config_id": ONLINE_CONFIG_ID, + "config_arn": ONLINE_CONFIG_ARN, + "custom_evaluator_ids": { + "HRResponseQuality": CUSTOM_RESPONSE_QUALITY_ID, + "HRSessionCompleteness": CUSTOM_SESSION_COMPLETENESS_ID, + }, + "evaluation_role_name": ONLINE_EVAL_ROLE_NAME, + "evaluation_role_arn": ONLINE_EVAL_ROLE_ARN, + "triggered_session_id": _online_session, + "results_log_group": f"/aws/bedrock-agentcore/evaluations/results/{ONLINE_CONFIG_ID}", + }, + indent=2, + ) +) +print(f"\n Config details saved: {_online_path}") + +# ============================================================ +# Summary +# ============================================================ + +print("\n" + "=" * 60) +print("Summary") +print("=" * 60) +print(" Custom evaluators created : HRResponseQuality, HRSessionCompleteness") +print(f" On-demand evaluation : {len(on_demand_results)} result(s) for session {SESSION_ID[:20]}...") +print(f" Online eval config : {ONLINE_EVAL_CONFIG_NAME} (ENABLED)") +print() +print(" Next steps:") +print(" - Check on-demand scores: results/on_demand_results.json") +print(" - Monitor online eval: AWS Console → CloudWatch → Log groups") +print(f" /aws/bedrock-agentcore/evaluations/results/{ONLINE_CONFIG_ID}") +print(" - Disable online config when done:") +print(" aws bedrock-agentcore-control update-online-evaluation-config \\") +print(f" --online-evaluation-config-id {ONLINE_CONFIG_ID} \\") +print(" --execution-status DISABLED") diff --git a/01-features/06-observe-evaluate-optimize-your-agent/02-evaluate/supported-frameworks/llamaindex/images/architecture.png b/01-features/06-observe-evaluate-optimize-your-agent/02-evaluate/supported-frameworks/llamaindex/images/architecture.png new file mode 100644 index 00000000..cf150e75 Binary files /dev/null and b/01-features/06-observe-evaluate-optimize-your-agent/02-evaluate/supported-frameworks/llamaindex/images/architecture.png differ diff --git a/01-features/06-observe-evaluate-optimize-your-agent/02-evaluate/supported-frameworks/llamaindex/images/sample-trace.png b/01-features/06-observe-evaluate-optimize-your-agent/02-evaluate/supported-frameworks/llamaindex/images/sample-trace.png new file mode 100644 index 00000000..27f58520 Binary files /dev/null and b/01-features/06-observe-evaluate-optimize-your-agent/02-evaluate/supported-frameworks/llamaindex/images/sample-trace.png differ diff --git a/01-features/06-observe-evaluate-optimize-your-agent/02-evaluate/supported-frameworks/llamaindex/llamaindex_hr_assistant.py b/01-features/06-observe-evaluate-optimize-your-agent/02-evaluate/supported-frameworks/llamaindex/llamaindex_hr_assistant.py new file mode 100644 index 00000000..b12f8cdf --- /dev/null +++ b/01-features/06-observe-evaluate-optimize-your-agent/02-evaluate/supported-frameworks/llamaindex/llamaindex_hr_assistant.py @@ -0,0 +1,392 @@ +""" +HR Assistant Agent: LlamaIndex agent workflow deployed on Bedrock AgentCore Runtime. + +Same HR Assistant domain as the shared Strands agent in ../../utils, re-implemented +as a LlamaIndex FunctionAgent workflow so it can be evaluated with AgentCore +Evaluations. The tools, mock data, and system prompt are identical, so ground-truth +and expected responses stay consistent across framework samples. + +Built as a LlamaIndex agent workflow (FunctionAgent) so the framework emits a +top-level workflow span with inference and tool child spans — the structure +AgentCore Evaluations reconstructs a session from. Tools are registered as +FunctionTool objects and return text-serializable values, per the AgentCore +best practices for LlamaIndex agents. + +Observability is provided by ADOT with the OpenTelemetry LlamaIndex instrumentation +(added to requirements.txt). ADOT discovers it at startup, so no explicit +instrumentation code is needed here. The LLM is a Bedrock model (Nova Lite) via +BedrockConverse, matching the shared Strands agent. + +Conversation history is persisted in AgentCore Memory (short-term memory +events) per runtime session, so multi-turn context survives microVM restarts. +deploy.py creates the memory resource and injects AGENTCORE_MEMORY_ID. History +is replayed via FunctionAgent's chat_history as plain USER/ASSISTANT text turns +(tool-call messages are not replayed — see the note above _load_chat_history). + +Tools (deterministic / mock data for reproducible evaluations): + get_pto_balance - remaining PTO days for an employee + submit_pto_request - request time off + lookup_hr_policy - company policy documents + get_benefits_summary - health, dental, vision, 401k, life insurance details + get_pay_stub - pay stub for a given period +""" + +import logging +import os +import re + +from bedrock_agentcore.memory import MemoryClient +from bedrock_agentcore.runtime import BedrockAgentCoreApp +from llama_index.core.agent.workflow import FunctionAgent +from llama_index.core.base.llms.types import ChatMessage +from llama_index.core.tools import FunctionTool +from llama_index.llms.bedrock_converse import BedrockConverse + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +app = BedrockAgentCoreApp() + +REGION = os.environ.get("AWS_REGION", "us-west-2") +MODEL_ID = os.environ.get("BEDROCK_MODEL_ID", "us.amazon.nova-lite-v1:0") + +# AgentCore Memory holds the conversation history across turns (and across +# microVM restarts). The memory resource is created by deploy.py and its id is +# injected as an environment variable on the runtime. +MEMORY_ID = os.environ.get("AGENTCORE_MEMORY_ID", "") +ACTOR_ID = "hr-employee" +_memory_client = MemoryClient(region_name=REGION) if MEMORY_ID else None + +# --------------------------------------------------------------------------- +# Mock data +# --------------------------------------------------------------------------- + +_PTO_BALANCES = { + "EMP-001": {"total_days": 15, "used_days": 5, "remaining_days": 10}, + "EMP-002": {"total_days": 15, "used_days": 12, "remaining_days": 3}, + "EMP-042": {"total_days": 20, "used_days": 7, "remaining_days": 13}, +} + +_HR_POLICIES = { + "pto": ( + "PTO Policy: Full-time employees accrue 15 days of PTO per year (20 days after 3 years). " + "PTO requests must be submitted at least 2 business days in advance. " + "Unused PTO up to 5 days rolls over to the next year. " + "PTO cannot be taken in advance of accrual." + ), + "remote_work": ( + "Remote Work Policy: Employees may work remotely up to 3 days per week with manager approval. " + "Core collaboration hours are 10am-3pm local time. " + "A dedicated workspace with reliable internet (25 Mbps+) is required. " + "Employees must be reachable via Slack and email during core hours." + ), + "parental_leave": ( + "Parental Leave Policy: Primary caregivers receive 16 weeks of fully paid parental leave. " + "Secondary caregivers receive 6 weeks of fully paid parental leave. " + "Leave may begin up to 2 weeks before the expected birth or adoption date. " + "Benefits continue unchanged during parental leave." + ), + "code_of_conduct": ( + "Code of Conduct: All employees are expected to treat colleagues, customers, and partners " + "with respect and professionalism. Harassment, discrimination, and retaliation of any kind " + "are strictly prohibited. Violations should be reported to HR or via the anonymous hotline." + ), +} + +_BENEFITS = { + "health": ( + "Health Insurance: The company covers 90% of premiums for employee-only coverage and 75% " + "for family coverage. Plans available: Blue Shield PPO, Kaiser HMO, and HDHP with HSA. " + "Annual deductible: $500 (PPO), $0 (HMO), $1,500 (HDHP). " + "Open enrollment is each November for the following calendar year." + ), + "dental": ( + "Dental Insurance: 100% coverage for preventive care (cleanings, X-rays). " + "80% coverage for basic restorative care (fillings, extractions). " + "50% coverage for major restorative care (crowns, bridges). " + "Annual maximum benefit: $2,000 per person. Orthodontia lifetime maximum: $1,500." + ), + "vision": ( + "Vision Insurance: Annual eye exam covered in full. " + "Frames or contacts allowance: $200 per year. " + "Laser vision correction discount: 15% off at participating providers." + ), + "401k": ( + "401(k) Plan: The company matches 100% of employee contributions up to 4% of salary. " + "An additional 50% match on the next 2% (total effective match up to 5%). " + "Employees are eligible to contribute immediately; company match vests over 3 years. " + "2026 IRS contribution limit: $23,500 (under 50), $31,000 (age 50+)." + ), + "life_insurance": ( + "Life Insurance: Basic life insurance of 2x annual salary provided at no cost. " + "Employees may purchase supplemental coverage up to 5x salary during open enrollment. " + "Accidental death and dismemberment (AD&D) coverage equal to basic life benefit is included." + ), +} + +_PAY_STUBS = { + ("EMP-001", "2025-12"): { + "gross_pay": 8333.33, + "federal_tax": 1458.33, + "state_tax": 416.67, + "social_security": 516.67, + "medicare": 120.83, + "health_premium": 125.00, + "401k_contribution": 333.33, + "net_pay": 5362.50, + "period": "December 2025", + }, + ("EMP-001", "2026-01"): { + "gross_pay": 8333.33, + "federal_tax": 1458.33, + "state_tax": 416.67, + "social_security": 516.67, + "medicare": 120.83, + "health_premium": 125.00, + "401k_contribution": 333.33, + "net_pay": 5362.50, + "period": "January 2026", + }, + ("EMP-042", "2026-01"): { + "gross_pay": 10416.67, + "federal_tax": 1875.00, + "state_tax": 520.83, + "social_security": 645.83, + "medicare": 151.04, + "health_premium": 200.00, + "401k_contribution": 416.67, + "net_pay": 6607.30, + "period": "January 2026", + }, +} + +_PTO_REQUEST_COUNTER = {"n": 0} + + +# --------------------------------------------------------------------------- +# Tool functions (registered as LlamaIndex FunctionTool objects below) +# --------------------------------------------------------------------------- + + +def get_pto_balance(employee_id: str) -> dict: + """ + Return the current PTO balance for an employee. + + Args: + employee_id: Employee identifier (e.g. EMP-001) + + Returns: + Dict with total_days, used_days, and remaining_days. + """ + balance = _PTO_BALANCES.get(employee_id) + if balance: + return {"employee_id": employee_id, **balance} + return {"employee_id": employee_id, "error": f"Employee {employee_id} not found."} + + +def submit_pto_request( + employee_id: str, + start_date: str, + end_date: str, + reason: str = "Personal time off", +) -> dict: + """ + Submit a PTO request for an employee. + + Args: + employee_id: Employee identifier (e.g. EMP-001) + start_date: First day of leave in YYYY-MM-DD format + end_date: Last day of leave in YYYY-MM-DD format + reason: Optional reason for the request + + Returns: + Dict with request_id, status, and confirmation message. + """ + _PTO_REQUEST_COUNTER["n"] += 1 + request_id = f"PTO-2026-{_PTO_REQUEST_COUNTER['n']:03d}" + return { + "request_id": request_id, + "employee_id": employee_id, + "start_date": start_date, + "end_date": end_date, + "reason": reason, + "status": "APPROVED", + "message": f"PTO request {request_id} approved for {employee_id} from {start_date} to {end_date}.", + } + + +def lookup_hr_policy(topic: str) -> dict: + """ + Look up a company HR policy document by topic. + + Args: + topic: Policy topic. Supported values: pto, remote_work, parental_leave, code_of_conduct + + Returns: + Dict with topic and policy_text. + """ + key = topic.lower().replace(" ", "_").replace("-", "_") + text = _HR_POLICIES.get(key) + if text: + return {"topic": topic, "policy_text": text} + return { + "topic": topic, + "error": f"Policy '{topic}' not found. Available: {list(_HR_POLICIES.keys())}", + } + + +def get_benefits_summary(benefit_type: str) -> dict: + """ + Return a summary of a specific employee benefit. + + Args: + benefit_type: Type of benefit. Supported values: health, dental, vision, 401k, life_insurance + + Returns: + Dict with benefit_type and summary text. + """ + key = benefit_type.lower().replace(" ", "_").replace("-", "_") + text = _BENEFITS.get(key) + if text: + return {"benefit_type": benefit_type, "summary": text} + return { + "benefit_type": benefit_type, + "error": f"Benefit '{benefit_type}' not found. Available: {list(_BENEFITS.keys())}", + } + + +def get_pay_stub(employee_id: str, period: str) -> dict: + """ + Retrieve a pay stub for an employee for a specific pay period. + + Args: + employee_id: Employee identifier (e.g. EMP-001) + period: Pay period in YYYY-MM format (e.g. 2026-01) + + Returns: + Dict with gross pay, deductions, and net pay. + """ + stub = _PAY_STUBS.get((employee_id, period)) + if stub: + return {"employee_id": employee_id, **stub} + return { + "employee_id": employee_id, + "period": period, + "error": f"Pay stub not found for {employee_id} period {period}.", + } + + +# --------------------------------------------------------------------------- +# Agent +# --------------------------------------------------------------------------- + +SYSTEM_PROMPT = """You are a helpful HR Assistant for Acme Corp. + +You help employees with: +- Checking PTO (paid time off) balances +- Submitting PTO requests +- Looking up HR policies (PTO, remote work, parental leave, code of conduct) +- Understanding employee benefits (health, dental, vision, 401k, life insurance) +- Retrieving pay stub information + +Always use the available tools to answer questions accurately. Do not make up +policy details, benefit amounts, or pay information. Look them up. +Be concise, professional, and friendly.""" + +_TOOLS = [ + FunctionTool.from_defaults(fn=get_pto_balance), + FunctionTool.from_defaults(fn=submit_pto_request), + FunctionTool.from_defaults(fn=lookup_hr_policy), + FunctionTool.from_defaults(fn=get_benefits_summary), + FunctionTool.from_defaults(fn=get_pay_stub), +] + +_AGENT = FunctionAgent( + tools=_TOOLS, + llm=BedrockConverse(model=MODEL_ID, region_name=REGION), + system_prompt=SYSTEM_PROMPT, + # Non-streaming: one complete inference span per model call is what the + # evaluation service reads, and it avoids a BedrockConverse streaming parser + # issue where split tool-call input deltas raise TypeError. + streaming=False, +) + +# Conversation history lives in AgentCore Memory (short-term memory events), +# keyed by the runtime session id. It survives microVM restarts and is shared +# with the AgentCore Memory console/APIs. +# +# Only USER/ASSISTANT text turns are stored and replayed as chat_history. +# Storing tool-call messages and replaying them breaks the Bedrock Converse +# API's toolUse/toolResult pairing validation, so intermediate tool messages +# are intentionally left out of memory. + + +def _load_chat_history(session_id: str) -> list: + """Load the conversation as ChatMessage items from AgentCore Memory.""" + if not _memory_client: + return [] + history = [] + events = _memory_client.list_events(memory_id=MEMORY_ID, actor_id=ACTOR_ID, session_id=session_id) + for event in sorted(events, key=lambda e: e["eventId"]): + for item in event.get("payload", []): + conv = item.get("conversational") + if conv: + role = "user" if conv["role"] == "USER" else "assistant" + history.append(ChatMessage(role=role, content=conv["content"]["text"])) + return history + + +def _save_turn(session_id: str, prompt: str, response: str): + """Persist one user/assistant turn to AgentCore Memory.""" + if not _memory_client: + return + _memory_client.create_event( + memory_id=MEMORY_ID, + actor_id=ACTOR_ID, + session_id=session_id, + messages=[(prompt, "USER"), (response, "ASSISTANT")], + ) + + +def _flush_telemetry(): + """ + Flush buffered OTel spans and event records before the microVM freezes. + + AgentCore Runtime suspends the microVM between invocations. Without an + explicit flush, event records buffered in the OTel batch processors (which + carry the agent's response text for evaluation) can be lost, and evaluators + then score empty responses. + """ + try: + from opentelemetry import trace as _trace + from opentelemetry._logs import get_logger_provider as _get_lp + + for provider in (_trace.get_tracer_provider(), _get_lp()): + flush = getattr(provider, "force_flush", None) + if flush: + flush() + except Exception: + logger.warning("Telemetry flush failed", exc_info=True) + + +@app.entrypoint +async def invoke(payload, context): + """Handle an agent invocation from AgentCore Runtime.""" + prompt = payload.get("prompt", "") + session_id = context.session_id or "default" + logger.info("Received prompt (session=%s): %s", session_id, prompt[:80]) + + try: + response = await _AGENT.run(prompt, chat_history=_load_chat_history(session_id)) + finally: + _flush_telemetry() + text = str(response) + # Strip inline ... blocks so spans and memory contain + # only the final answer + text = re.sub(r".*?", "", text, flags=re.DOTALL).strip() + _save_turn(session_id, prompt, text) + return text + + +if __name__ == "__main__": + app.run() diff --git a/01-features/06-observe-evaluate-optimize-your-agent/02-evaluate/supported-frameworks/llamaindex/requirements.txt b/01-features/06-observe-evaluate-optimize-your-agent/02-evaluate/supported-frameworks/llamaindex/requirements.txt new file mode 100644 index 00000000..d8fee7b9 --- /dev/null +++ b/01-features/06-observe-evaluate-optimize-your-agent/02-evaluate/supported-frameworks/llamaindex/requirements.txt @@ -0,0 +1,2 @@ +bedrock-agentcore>=1.6.0 +boto3>=1.43.0 diff --git a/01-features/06-observe-evaluate-optimize-your-agent/02-evaluate/supported-frameworks/openai-agents/.gitignore b/01-features/06-observe-evaluate-optimize-your-agent/02-evaluate/supported-frameworks/openai-agents/.gitignore new file mode 100644 index 00000000..1b41b541 --- /dev/null +++ b/01-features/06-observe-evaluate-optimize-your-agent/02-evaluate/supported-frameworks/openai-agents/.gitignore @@ -0,0 +1,3 @@ +results/ +__pycache__/ +agent_config.json diff --git a/01-features/06-observe-evaluate-optimize-your-agent/02-evaluate/supported-frameworks/openai-agents/README.md b/01-features/06-observe-evaluate-optimize-your-agent/02-evaluate/supported-frameworks/openai-agents/README.md new file mode 100644 index 00000000..1adbf692 --- /dev/null +++ b/01-features/06-observe-evaluate-optimize-your-agent/02-evaluate/supported-frameworks/openai-agents/README.md @@ -0,0 +1,171 @@ +# Evaluate an OpenAI Agents SDK agent + +Evaluate an [OpenAI Agents SDK](https://openai.github.io/openai-agents-python/) agent with Amazon Bedrock AgentCore Evaluations. This sample deploys the shared HR Assistant, re-implemented with the OpenAI Agents SDK, to AgentCore Runtime. It then scores the agent with built-in and custom LLM-as-a-judge evaluators in on-demand and online modes. See [OpenAI Agents support in AgentCore Evaluations](https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/supported-frameworks-openai-agents.html) for the supported instrumentation libraries, scope names, and span extraction rules. + +The HR Assistant, its 5 tools, mock data, and system prompt are identical to the Strands version in [`../../utils/`](../../utils/), so ground-truth and expected responses stay consistent across the framework samples. + +## What you'll learn + +| Concept | Description | +| ----------------------------- | ------------------------------------------------------------------------------------------------------- | +| Framework instrumentation | Make an OpenAI Agents SDK agent evaluable by adding one OpenTelemetry package without instrumentation code | +| OpenAI GPT-5.5 on Bedrock | Call `openai.gpt-5.5` through Bedrock's OpenAI-compatible Responses API with a Bedrock API key | +| AgentCore Memory | Persist multi-turn conversation history in the AgentCore Memory service, across microVM restarts | +| On-demand evaluation | Score a recorded session with built-in + custom LLM-as-a-judge evaluators via `EvaluationClient` | +| Online evaluation | Continuously score live traffic with an online evaluation config | +| CLI evaluation | Re-evaluate any session from the terminal with the AgentCore CLI | + +## Architecture + +![OpenAI Agents evaluation flow across AgentCore Runtime, CloudWatch, and Evaluations](images/architecture.png) + +The PNG embeds its draw.io XML and can be opened directly in draw.io for editing. + +## How it works + +The agent is instrumented for evaluation with the OpenTelemetry OpenAI Agents library (`opentelemetry-instrumentation-openai-agents`, scope `opentelemetry.instrumentation.openai_agents`). On AgentCore Runtime, AWS Distro for OpenTelemetry (ADOT) auto-discovers the library at startup, so no explicit instrumentation code is needed. The agent's spans and event records flow to CloudWatch, and AgentCore Evaluations reads them from there. + +The LLM is OpenAI GPT-5.5 on Amazon Bedrock (`openai.gpt-5.5`), reached through the Bedrock mantle endpoint's OpenAI-compatible Responses API and authenticated with a [Bedrock API key](https://docs.aws.amazon.com/bedrock/latest/userguide/api-keys.html). The OpenAI Agents SDK talks to it via an `AsyncOpenAI` client: + +```python +from agents import Agent, OpenAIResponsesModel +from openai import AsyncOpenAI +from aws_bedrock_token_generator import provide_token + +# Short-term Bedrock API key minted from the runtime's IAM role by a local +# SigV4 presign, no network call, nothing stored in code or config +api_key = provide_token(region=MODEL_REGION) +client = AsyncOpenAI(base_url=f"https://bedrock-mantle.{MODEL_REGION}.api.aws/openai/v1", api_key=api_key) +model = OpenAIResponsesModel(model="openai.gpt-5.5", openai_client=client) + +agent = Agent(name="HRAssistant", instructions=SYSTEM_PROMPT, model=model, tools=[...]) +``` + +`provide_token()` returns a [short-term Bedrock API key](https://docs.aws.amazon.com/bedrock/latest/userguide/api-keys-how.html) (a `bedrock-api-key-...` string) with the same permissions as the runtime's IAM role, valid up to 12 hours. This is the secure kind AWS recommends for production. The agent is rebuilt on every invocation so a long-lived runtime never reuses an expired key. For exploration, you can instead generate a long-term key (Bedrock console → API keys, or `aws iam create-service-specific-credential --service-name bedrock.amazonaws.com`) and pass it as the `api_key`. Store it in AWS Secrets Manager rather than in code. + +Three implementation details matter for evaluation: + +- Responses API, not Chat Completions. The OpenTelemetry instrumentation extracts the agent's response text from Responses API spans (`ResponseSpanData`); with `OpenAIChatCompletionsModel` the response text is not captured on the spans and evaluators score empty responses. GPT-5.5 is served on the mantle endpoint's `openai/v1` path (`https://bedrock-mantle..api.aws/openai/v1`). This differs from the `/v1` path used by gpt-oss models. +- Keep SDK tracing enabled. The instrumentation hooks into the SDK's tracing pipeline, so do not call `set_tracing_disabled(True)`. Doing so would silence the evaluation spans. The SDK's default platform.openai.com exporter is inert without an `OPENAI_API_KEY` and only logs a skip message. +- AgentCore Memory for conversation history, not `SQLiteSession`. The SDK's session classes are local to one microVM (history is lost across restarts) and replay full Responses API output items (including model `reasoning` items) as the next turn's input, which the mantle endpoint rejects with an empty output. Instead, `deploy.py` creates an [AgentCore Memory](https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/memory.html) resource and the agent persists each turn as a memory event (via `bedrock_agentcore.memory.MemoryClient`), reloading the plain `{"role", "content"}` history at the start of every invocation. + +The agent is rebuilt on every invocation so a long-lived runtime never reuses an expired short-term key. + +## Sample trace + +![Sanitized OpenAI Agents trace showing agent, model, and tool spans](images/sample-trace.png) + +This trace was captured from the deployed sample and sanitized for publication. AgentCore Evaluations identifies `HRAssistant.agent` as the agent invocation from `gen_ai.operation.name=invoke_agent`, the two `openai.response` spans as model calls from `gen_ai.operation.name=chat`, and `get_pto_balance.tool` as the tool call from `gen_ai.operation.name=execute_tool`. The two chat spans are separate model calls before and after tool execution. The shared scope is `opentelemetry.instrumentation.openai_agents`. The PNG embeds its draw.io XML and can be opened directly in draw.io for editing. + +## Prerequisites + +- Python 3.10+ +- AWS CLI configured with credentials +- Access to `openai.gpt-5.5` on Amazon Bedrock. GPT-5.5 is served from `us-east-1` / `us-east-2` (see [the model card](https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html)); the runtime can be deployed in any region and calls the model cross-region via `BEDROCK_OPENAI_MODEL_REGION` (default `us-east-1`). +- Permissions for: `bedrock-agentcore:*`, `bedrock-agentcore-control:*`, `logs:*`, `iam:CreateRole`, `iam:PutRolePolicy`, `s3:PutObject`, `bedrock:InvokeModel` + +## Deploy the agent + +```bash +uv run --frozen --with-requirements requirements.txt python deploy.py --region us-west-2 +``` + +This builds an ARM64 deployment package, creates an AgentCore Memory resource (conversation history store, injected as `AGENTCORE_MEMORY_ID`), creates the AgentCore Runtime, and writes `agent_config.json` in this directory (read by `evaluate.py`). + +The runtime's IAM role is granted `bedrock-mantle:CreateInference` and `bedrock-mantle:CallWithBearerToken` for the mantle Responses API, plus `bedrock:InvokeModel*` and `bedrock:CallWithBearerToken` (the latter pair covers the `bedrock-runtime/openai/v1` Chat Completions endpoint, should you switch `BEDROCK_OPENAI_BASE_URL` to it). The bearer-token actions are required because `provide_token()`'s short-term API key authenticates against these endpoints with `bedrock:CallWithBearerToken` rather than SigV4. + +## Run the evaluation + +```bash +uv run --frozen --with-requirements requirements.txt python evaluate.py --region us-west-2 +``` + +The script: + +1. Creates two custom LLM-as-a-judge evaluators (`HRResponseQuality` TRACE, `HRSessionCompleteness` SESSION). +2. Invokes the deployed agent for a 3-turn session and waits ~90s for CloudWatch span ingestion. +3. Runs on-demand evaluation with `EvaluationClient` (built-in + custom evaluators, with `ReferenceInputs` ground truth). Scores are saved to `results/on_demand_results.json`. +4. Creates an online evaluation config that continuously scores live traffic with built-in evaluators. Details are saved to `results/online_eval_config.json`. + +## Expected output + +``` +[1/4] Creating custom LLM-as-a-judge evaluators ... + Creating HRResponseQuality (TRACE) ... + Creating HRSessionCompleteness (SESSION) ... + +[2/4] Invoking HR Assistant to generate a session ... + Turn 1: What is the PTO balance for employee EMP-001? + -> Employee EMP-001 has 10 PTO days remaining (15 total, 5 used) ... + Turn 2: Please submit a PTO request for EMP-001 from 2026-07-14 to 2026-07-18. + -> Your PTO request has been submitted and approved. Request ID: PTO-2026-001 ... + Turn 3: What is the company remote work policy? + -> Employees may work remotely up to 3 days per week ... + +[3/4] Running on-demand evaluation (EvaluationClient) ... + Evaluator Value Label + -------------------------------------------------------------------------------- + Builtin.GoalSuccessRate 1.0 Yes + Builtin.Correctness 1.0 Perfectly Correct + Builtin.Helpfulness 1.0 Above And Beyond + HRResponseQuality 1.0 excellent + HRSessionCompleteness 1.0 complete + +[4/4] Creating online evaluation configuration ... + Online evaluation config created: hr_openai_eval_-XXXXXXXXXX +``` + +TRACE-level evaluators (`Correctness`, `Helpfulness`, `HRResponseQuality`) return one score per turn, so the full run prints 11 results. Online evaluation results appear a few minutes later in CloudWatch at `/aws/bedrock-agentcore/evaluations/results/`, one record per evaluator per sampled turn with `gen_ai.evaluation.score.value` and `gen_ai.evaluation.explanation` attributes. + +## Evaluate from the CLI + +Once sessions exist in CloudWatch, you can re-evaluate them from the terminal with the [AgentCore CLI](https://www.npmjs.com/package/@aws/agentcore). No Python is needed. Because this sample deploys with a plain `deploy.py` (not an `agentcore` project), use the standalone flags: + +```bash +npm install -g @aws/agentcore + +AGENT_ARN=$(jq -r .agent_arn agent_config.json) +agentcore run eval \ + --runtime-arn "$AGENT_ARN" \ + --evaluator-arn Builtin.Helpfulness Builtin.Correctness \ + --region us-west-2 \ + --session-id \ + --days 1 +``` + +``` +Agent: hr_openai_xxxxxxxx-XXXXXXXXXX | Sessions: 1 | Lookback: 1d + + Builtin.Helpfulness: 0.94 + +Results saved to: eval_2026-07-10_13-03-07.json +``` + +Ground truth can be supplied inline with `--assertion`, `--expected-trajectory`, and `--expected-response`. Omit `--session-id` to evaluate every session in the lookback window. + +## Troubleshooting ARM64 wheels + +`deploy.py` cross-compiles dependencies with `--platform manylinux2014_aarch64 --only-binary=:all:`. If a dependency lacks an aarch64 wheel and the install fails, either: + +- add `--no-binary=` for the offending pure-Python package, or +- build the zip on an ARM64 machine or in a `public.ecr.aws/lambda/python:3.13-arm64` container / AWS CodeBuild ARM instead of cross-compiling. + +## Clean up + +Run the cleanup script from this directory: + +```bash +uv run --frozen --with-requirements requirements.txt python cleanup.py +``` + +The script uses the `default` AWS profile and the region in `agent_config.json`. It deletes the online evaluation configurations and custom evaluators recorded under `results/`, then removes the AgentCore Runtime, Memory, sample-specific CloudWatch log groups, deployment package, and IAM roles. Asynchronous AgentCore deletions are checked for completion before dependent resources are removed, and the script can be run again if cleanup is interrupted. + +The shared `aws/spans` log group is retained. The regional deployment bucket is also retained when it contains objects from other samples. Use `--profile` or `--region` to override the defaults. + +## Additional resources + +- [Supported agent frameworks: OpenAI Agents](https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/supported-frameworks-openai-agents.html) +- [OpenAI Agents SDK](https://openai.github.io/openai-agents-python/) +- [GPT-5.5 model card](https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-55.html) +- [Amazon Bedrock API keys](https://docs.aws.amazon.com/bedrock/latest/userguide/api-keys.html) +- [Amazon Bedrock AgentCore Developer Guide](https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/) diff --git a/01-features/06-observe-evaluate-optimize-your-agent/02-evaluate/supported-frameworks/openai-agents/cleanup.py b/01-features/06-observe-evaluate-optimize-your-agent/02-evaluate/supported-frameworks/openai-agents/cleanup.py new file mode 100644 index 00000000..8ce11e73 --- /dev/null +++ b/01-features/06-observe-evaluate-optimize-your-agent/02-evaluate/supported-frameworks/openai-agents/cleanup.py @@ -0,0 +1,435 @@ +"""Delete AWS resources created by the OpenAI Agents sample. + +The script reads agent_config.json plus optional files in results/ and removes +the sample-specific evaluation, runtime, memory, logging, S3, and IAM resources. + +Usage: + uv run --frozen --with-requirements requirements.txt python cleanup.py + [--region REGION] [--profile PROFILE] +""" + +from __future__ import annotations + +import argparse +import json +import sys +import time +from collections.abc import Callable, Sequence +from functools import partial +from pathlib import Path +from typing import Any, TypedDict + +import boto3 +from botocore.config import Config +from botocore.exceptions import BotoCoreError, ClientError + +_SCRIPT_DIR = Path(__file__).parent +_DEFAULT_CONFIG = _SCRIPT_DIR / "agent_config.json" +_DEFAULT_RESULTS_DIR = _SCRIPT_DIR / "results" +_NOT_FOUND_CODES = { + "404", + "NoSuchBucket", + "NoSuchEntity", + "NotFoundException", + "ResourceNotFoundException", +} +_Action = Callable[[], object] +_GetAction = Callable[[], dict[str, Any]] +_StatusGetter = Callable[[dict[str, Any]], str | None] + + +class EvaluationState(TypedDict): + """Resource identifiers written by evaluate.py.""" + + online_config_ids: list[str] + custom_evaluator_ids: list[str] + evaluation_role_names: list[str] + results_log_groups: list[str] + + +def _read_json(path: Path) -> dict[str, Any]: + """Read a JSON object from path. + + Args: + path: JSON file to read. + + Returns: + The decoded object, or an empty dictionary when the file is absent. + + Raises: + ValueError: If the JSON root is not an object. + """ + if not path.exists(): + return {} + value = json.loads(path.read_text()) + if not isinstance(value, dict): + raise ValueError(f"Expected a JSON object in {path}") + return value + + +def _collect_evaluation_state(results_dir: Path) -> EvaluationState: + """Merge cleanup identifiers from evaluation result files.""" + online_config_ids: set[str] = set() + evaluator_ids: set[str] = set() + evaluation_role_names: set[str] = set() + results_log_groups: set[str] = set() + + for filename in ("on_demand_results.json", "online_eval_config.json", "cleanup_state.json"): + data = _read_json(results_dir / filename) + _collect_strings(data.get("online_evaluation_config_id"), online_config_ids) + _collect_strings(data.get("online_evaluation_config_ids"), online_config_ids) + _collect_strings(data.get("config_id"), online_config_ids) + _collect_strings(data.get("custom_evaluator_ids"), evaluator_ids, exclude_builtins=True) + _collect_strings(data.get("evaluation_role_name"), evaluation_role_names) + _collect_strings(data.get("evaluation_role_names"), evaluation_role_names) + _collect_strings(data.get("results_log_group"), results_log_groups) + _collect_strings(data.get("results_log_groups"), results_log_groups) + + return { + "online_config_ids": sorted(online_config_ids), + "custom_evaluator_ids": sorted(evaluator_ids), + "evaluation_role_names": sorted(evaluation_role_names), + "results_log_groups": sorted(results_log_groups), + } + + +def _collect_strings(value: object, destination: set[str], *, exclude_builtins: bool = False) -> None: + """Collect strings from a scalar, list, or dictionary value.""" + candidates: Sequence[object] + if isinstance(value, str): + candidates = [value] + elif isinstance(value, list): + candidates = value + elif isinstance(value, dict): + candidates = list(value.values()) + else: + return + + destination.update( + candidate + for candidate in candidates + if isinstance(candidate, str) and candidate and (not exclude_builtins or not candidate.startswith("Builtin.")) + ) + + +def _role_name_from_arn(role_arn: str) -> str | None: + """Return the IAM role name from an ARN.""" + marker = ":role/" + if marker not in role_arn: + return None + return role_arn.split(marker, 1)[1].rsplit("/", 1)[-1] + + +def _error_code(error: ClientError) -> str: + """Return an AWS service error code.""" + return str(error.response.get("Error", {}).get("Code", "Unknown")) + + +def _run_step(label: str, action: _Action, failures: list[str]) -> bool: + """Run one cleanup action while treating missing resources as success.""" + print(f"Deleting {label} ...") + try: + action() + print(" [ok]") + return True + except ClientError as error: + code = _error_code(error) + if code in _NOT_FOUND_CODES: + print(" [skip] already absent") + return True + failures.append(f"{label}: {code}: {error}") + print(f" [failed] {code}: {error}") + return False + except (BotoCoreError, OSError, RuntimeError, TimeoutError) as error: + failures.append(f"{label}: {error}") + print(f" [failed] {error}") + return False + + +def _flat_status(response: dict[str, Any]) -> str | None: + """Read a top-level AgentCore resource status.""" + status = response.get("status") + return status if isinstance(status, str) else None + + +def _memory_status(response: dict[str, Any]) -> str | None: + """Read an AgentCore Memory status.""" + memory = response.get("memory") + if not isinstance(memory, dict): + return None + status = memory.get("status") + return status if isinstance(status, str) else None + + +def _delete_async_resource( + label: str, + delete_action: _Action, + get_action: _GetAction, + get_status: _StatusGetter, + failures: list[str], + *, + poll_interval: float, + timeout: float, +) -> bool: + """Delete an AgentCore resource and wait until it no longer exists.""" + print(f"Deleting {label} ...") + try: + try: + response = get_action() + except ClientError as error: + if _error_code(error) in _NOT_FOUND_CODES: + print(" [skip] already absent") + return True + raise + + if get_status(response) != "DELETING": + try: + delete_action() + except ClientError as error: + code = _error_code(error) + if code in _NOT_FOUND_CODES: + print(" [ok]") + return True + if code != "ConflictException": + raise + response = get_action() + if get_status(response) != "DELETING": + raise RuntimeError( + f"delete request conflicted; current status is {get_status(response) or 'unknown'}" + ) from error + + deadline = time.monotonic() + timeout + while True: + try: + response = get_action() + except ClientError as error: + if _error_code(error) in _NOT_FOUND_CODES: + print(" [ok]") + return True + raise + + status = get_status(response) + if time.monotonic() >= deadline: + raise TimeoutError( + f"deletion did not finish within {timeout:g} seconds; current status is {status or 'unknown'}" + ) + time.sleep(poll_interval) + except ClientError as error: + code = _error_code(error) + failures.append(f"{label}: {code}: {error}") + print(f" [failed] {code}: {error}") + return False + except (BotoCoreError, RuntimeError, TimeoutError) as error: + failures.append(f"{label}: {error}") + print(f" [failed] {error}") + return False + + +def _delete_iam_role(iam: Any, role_name: str) -> None: + """Delete all policies from an IAM role, then delete the role.""" + inline_pages = iam.get_paginator("list_role_policies").paginate(RoleName=role_name) + for page in inline_pages: + for policy_name in page.get("PolicyNames", []): + iam.delete_role_policy(RoleName=role_name, PolicyName=policy_name) + + attached_pages = iam.get_paginator("list_attached_role_policies").paginate(RoleName=role_name) + for page in attached_pages: + for policy in page.get("AttachedPolicies", []): + policy_arn = policy.get("PolicyArn") + if policy_arn: + iam.detach_role_policy(RoleName=role_name, PolicyArn=policy_arn) + + iam.delete_role(RoleName=role_name) + + +def _delete_empty_bucket(s3: Any, bucket: str) -> None: + """Delete the code bucket only when no other sample artifacts remain.""" + response = s3.list_objects_v2(Bucket=bucket, MaxKeys=1) + if response.get("KeyCount", 0): + print(" Bucket contains other objects and will be retained.") + return + s3.delete_bucket(Bucket=bucket) + + +def _require_string(config: dict[str, Any], key: str) -> str: + """Return a required non-empty string from configuration.""" + value = config.get(key) + if not isinstance(value, str) or not value: + raise ValueError(f"Missing required '{key}' in agent config") + return value + + +def _parse_args(argv: Sequence[str] | None) -> argparse.Namespace: + """Parse command-line arguments.""" + parser = argparse.ArgumentParser(description="Clean up the OpenAI Agents evaluation sample") + parser.add_argument("--config", type=Path, default=_DEFAULT_CONFIG, help="Path to agent_config.json") + parser.add_argument("--results-dir", type=Path, default=_DEFAULT_RESULTS_DIR, help="Evaluation results directory") + parser.add_argument("--region", default=None, help="Override the region saved by deploy.py") + parser.add_argument("--profile", default="default", help="AWS profile (default: default)") + parser.add_argument( + "--poll-interval", type=_non_negative_float, default=5.0, help="Seconds between deletion checks" + ) + parser.add_argument( + "--timeout", + type=_non_negative_float, + default=300.0, + help="Seconds to wait for each asynchronous deletion", + ) + return parser.parse_args(argv) + + +def _non_negative_float(value: str) -> float: + """Parse a non-negative command-line number.""" + parsed = float(value) + if parsed < 0: + raise argparse.ArgumentTypeError("must be zero or greater") + return parsed + + +def main(argv: Sequence[str] | None = None) -> int: + """Run cleanup and return a process exit code.""" + args = _parse_args(argv) + try: + config = _read_json(args.config) + if not config: + raise ValueError(f"Agent config not found: {args.config}. Run deploy.py first.") + + region = args.region or _require_string(config, "region") + agent_id = _require_string(config, "agent_id") + evaluation = _collect_evaluation_state(args.results_dir) + + session = boto3.Session(profile_name=args.profile, region_name=region) + client_config = Config(retries={"mode": "adaptive", "total_max_attempts": 5}) + identity = session.client("sts", config=client_config).get_caller_identity() + control = session.client("bedrock-agentcore-control", config=client_config) + logs = session.client("logs", config=client_config) + iam = session.client("iam", config=client_config) + s3 = session.client("s3", config=client_config) + + print(f"AWS account: {identity['Account']}") + print(f"Region: {region}") + print(f"Runtime: {agent_id}") + print() + + failures: list[str] = [] + + online_configs_deleted = True + for online_config_id in evaluation["online_config_ids"]: + deleted = _delete_async_resource( + f"online evaluation config {online_config_id}", + lambda: control.delete_online_evaluation_config( + onlineEvaluationConfigId=online_config_id, + ), + lambda: control.get_online_evaluation_config( + onlineEvaluationConfigId=online_config_id, + ), + _flat_status, + failures, + poll_interval=args.poll_interval, + timeout=args.timeout, + ) + online_configs_deleted = deleted and online_configs_deleted + + if online_configs_deleted: + for evaluator_id in evaluation["custom_evaluator_ids"]: + _delete_async_resource( + f"custom evaluator {evaluator_id}", + partial(control.delete_evaluator, evaluatorId=evaluator_id), + partial(control.get_evaluator, evaluatorId=evaluator_id), + _flat_status, + failures, + poll_interval=args.poll_interval, + timeout=args.timeout, + ) + else: + print("Skipping evaluators and dependent resources because an online evaluation config remains.") + + runtime_deleted = False + if online_configs_deleted: + runtime_deleted = _delete_async_resource( + f"AgentCore Runtime {agent_id}", + lambda: control.delete_agent_runtime(agentRuntimeId=agent_id), + lambda: control.get_agent_runtime(agentRuntimeId=agent_id), + _flat_status, + failures, + poll_interval=args.poll_interval, + timeout=args.timeout, + ) + else: + print("Skipping AgentCore Runtime because an online evaluation config remains.") + + memory_id = config.get("memory_id") + if runtime_deleted and isinstance(memory_id, str) and memory_id: + _delete_async_resource( + f"AgentCore Memory {memory_id}", + lambda: control.delete_memory(memoryId=memory_id), + lambda: control.get_memory(memoryId=memory_id), + _memory_status, + failures, + poll_interval=args.poll_interval, + timeout=args.timeout, + ) + + runtime_log_group = config.get("cw_log_group") + if runtime_deleted and isinstance(runtime_log_group, str) and runtime_log_group: + _run_step( + f"runtime log group {runtime_log_group}", + lambda: logs.delete_log_group(logGroupName=runtime_log_group), + failures, + ) + + if online_configs_deleted: + for results_log_group in evaluation["results_log_groups"]: + _run_step( + f"evaluation results log group {results_log_group}", + partial(logs.delete_log_group, logGroupName=results_log_group), + failures, + ) + + s3_bucket = config.get("s3_bucket") + s3_key = config.get("s3_key") + if runtime_deleted and isinstance(s3_bucket, str) and s3_bucket and isinstance(s3_key, str) and s3_key: + _run_step( + f"S3 object s3://{s3_bucket}/{s3_key}", + lambda: s3.delete_object(Bucket=s3_bucket, Key=s3_key), + failures, + ) + _run_step( + f"empty S3 bucket {s3_bucket}", + lambda: _delete_empty_bucket(s3, s3_bucket), + failures, + ) + + if online_configs_deleted: + for evaluation_role_name in evaluation["evaluation_role_names"]: + _run_step( + f"evaluation IAM role {evaluation_role_name}", + partial(_delete_iam_role, iam, evaluation_role_name), + failures, + ) + + role_arn = config.get("role_arn") + runtime_role_name = _role_name_from_arn(role_arn) if isinstance(role_arn, str) else None + if runtime_deleted and runtime_role_name: + _run_step( + f"runtime IAM role {runtime_role_name}", + lambda: _delete_iam_role(iam, runtime_role_name), + failures, + ) + + if failures: + print("\nCleanup finished with failures:") + for failure in failures: + print(f" - {failure}") + print("Re-run cleanup.py after resolving the reported errors.") + return 1 + + print("\nCleanup complete.") + return 0 + except (BotoCoreError, ClientError, OSError, ValueError, json.JSONDecodeError) as error: + print(f"Cleanup failed: {error}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/01-features/06-observe-evaluate-optimize-your-agent/02-evaluate/supported-frameworks/openai-agents/deploy.py b/01-features/06-observe-evaluate-optimize-your-agent/02-evaluate/supported-frameworks/openai-agents/deploy.py new file mode 100644 index 00000000..702e9f4f --- /dev/null +++ b/01-features/06-observe-evaluate-optimize-your-agent/02-evaluate/supported-frameworks/openai-agents/deploy.py @@ -0,0 +1,305 @@ +"""Deploy the OpenAI Agents HR Assistant to AgentCore Runtime using the bedrock-agentcore SDK. + +Packages the agent source and its dependencies into a zip, uploads to S3, creates an +AgentCore Runtime, and polls until READY. Saves connection details to agent_config.json +in this directory for use by evaluate.py. + +Usage: + python deploy.py [--region REGION] + +Output: + agent_config.json — AGENT_ID, AGENT_ARN, CW_LOG_GROUP, REGION + +Deployment steps: + 1. Create an IAM execution role for the runtime + 2. Create an AgentCore Memory resource (conversation history store) + 3. Package openai_hr_assistant.py + ARM64 dependencies into a zip + 4. Upload the zip to S3 + 5. Create an AgentCore Runtime via create_agent_runtime (codeConfiguration), + injecting AGENTCORE_MEMORY_ID as an environment variable + 6. Poll until READY + 7. Write agent_config.json + +The runtime uses OpenAI GPT-5.5 on Bedrock via the mantle endpoint's OpenAI +Responses API, authenticated with a Bedrock API key (short-term by default, +minted from the runtime role; long-term via the BEDROCK_API_KEY env var). +The role policy below grants the required bedrock-mantle:CreateInference and +bedrock-mantle:CallWithBearerToken actions (plus bedrock:CallWithBearerToken for +the bedrock-runtime /openai/v1 Chat Completions alternative). GPT-5.5 is served +from us-east-1/us-east-2; the agent calls it cross-region from wherever the +runtime is deployed. + +See https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/getting-started-custom.html +""" + +import argparse +import json +import os +import shutil +import subprocess +import sys +import time +import uuid +import zipfile +from pathlib import Path + +import boto3 +from boto3.session import Session + +# --------------------------------------------------------------------------- +# Config +# --------------------------------------------------------------------------- + +_SCRIPT_DIR = Path(__file__).parent +_CONFIG_FILE = _SCRIPT_DIR / "agent_config.json" +_AGENT_FILE = "openai_hr_assistant.py" + +# Bundled into the deployment zip (ARM64). openai-agents talks to the Bedrock +# OpenAI-compatible endpoint; aws-bedrock-token-generator mints the bearer token; +# the OpenTelemetry instrumentation is auto-discovered by ADOT at startup. +_PACKAGES = [ + "openai-agents", + "openai", + "opentelemetry-instrumentation-openai-agents>=0.61.0", + "aws-bedrock-token-generator", + "bedrock-agentcore", + "aws-opentelemetry-distro", +] + +parser = argparse.ArgumentParser(description="Deploy the OpenAI Agents HR Assistant to AgentCore Runtime") +parser.add_argument("--region", default=None, help="AWS region (default: boto3 session region)") +args = parser.parse_args() + +REGION = args.region or Session().region_name or "us-west-2" +print(f"Region: {REGION}") + +_sts = boto3.client("sts", region_name=REGION) +_ACCOUNT_ID = _sts.get_caller_identity()["Account"] +_iam = boto3.client("iam", region_name=REGION) +_s3 = boto3.client("s3", region_name=REGION) +_ctrl = boto3.client("bedrock-agentcore-control", region_name=REGION) + +_AGENT_NAME = f"hr_openai_{uuid.uuid4().hex[:8]}" +_ROLE_NAME = f"{_AGENT_NAME}_role" +_S3_BUCKET = f"bedrock-agentcore-code-{_ACCOUNT_ID}-{REGION}" +_S3_KEY = f"{_AGENT_NAME}/deployment_package.zip" +_BUILD_DIR = Path(f"/tmp/{_AGENT_NAME}_build") # nosec B108 + +# --------------------------------------------------------------------------- +# 1. IAM execution role +# --------------------------------------------------------------------------- + +_TRUST = json.dumps( + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": {"Service": "bedrock-agentcore.amazonaws.com"}, + "Action": "sts:AssumeRole", + "Condition": { + "StringEquals": {"aws:SourceAccount": _ACCOUNT_ID}, + "ArnLike": {"aws:SourceArn": f"arn:aws:bedrock-agentcore:*:{_ACCOUNT_ID}:runtime/*"}, + }, + } + ], + } +) + +_POLICY = json.dumps( + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "bedrock:InvokeModel", + "bedrock:InvokeModelWithResponseStream", + "bedrock:CallWithBearerToken", + "bedrock-mantle:CreateInference", + "bedrock-mantle:CallWithBearerToken", + "bedrock-agentcore:CreateEvent", + "bedrock-agentcore:ListEvents", + "bedrock-agentcore:GetEvent", + "bedrock-agentcore:GetMemory", + "bedrock-agentcore:ListMemoryRecords", + "bedrock-agentcore:RetrieveMemoryRecords", + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:PutLogEvents", + "logs:DescribeLogGroups", + "logs:DescribeLogStreams", + "xray:PutTraceSegments", + "xray:PutTelemetryRecords", + "xray:GetSamplingRules", + "xray:GetSamplingTargets", + "cloudwatch:PutMetricData", + ], + "Resource": "*", + } + ], + } +) + +print(f"\n[1/5] Creating IAM role '{_ROLE_NAME}' ...") +try: + _ROLE_ARN = _iam.create_role(RoleName=_ROLE_NAME, AssumeRolePolicyDocument=_TRUST)["Role"]["Arn"] + print(f" Created: {_ROLE_ARN}") +except _iam.exceptions.EntityAlreadyExistsException: + _ROLE_ARN = _iam.get_role(RoleName=_ROLE_NAME)["Role"]["Arn"] + print(f" Already exists: {_ROLE_ARN}") + +_iam.put_role_policy( + RoleName=_ROLE_NAME, + PolicyName=f"{_AGENT_NAME}_policy", + PolicyDocument=_POLICY, +) +print(" Policy attached. Waiting 10s for IAM propagation ...") +time.sleep(10) + +# --------------------------------------------------------------------------- +# 2. Create AgentCore Memory (conversation history store) +# --------------------------------------------------------------------------- + +print(f"\n[2/6] Creating AgentCore Memory '{_AGENT_NAME}_memory' ...") +_mem_resp = _ctrl.create_memory( + name=f"{_AGENT_NAME}_memory", + description="Short-term conversation memory for the HR Assistant sample", + eventExpiryDuration=7, +) +MEMORY_ID = _mem_resp["memory"]["id"] +print(f" Memory ID: {MEMORY_ID}") +for _elapsed in range(0, 300, 10): + _mstatus = _ctrl.get_memory(memoryId=MEMORY_ID)["memory"]["status"] + if _mstatus == "ACTIVE": + break + if _mstatus == "FAILED": + raise RuntimeError("Memory creation failed") + time.sleep(10) +print(f" Memory status: {_mstatus}") + +# --------------------------------------------------------------------------- +# 3. Build deployment package (ARM64) +# --------------------------------------------------------------------------- + +print("\n[3/6] Building deployment package ...") +if _BUILD_DIR.exists(): + shutil.rmtree(_BUILD_DIR) +_PKG = _BUILD_DIR / "pkg" +_PKG.mkdir(parents=True) + +subprocess.run( + [ + sys.executable, + "-m", + "pip", + "install", + *_PACKAGES, + "-t", + str(_PKG), + "--platform", + "manylinux2014_aarch64", + "--only-binary=:all:", + "--python-version", + "3.13", + # This is an isolated --target install; suppress pip's dependency check + # against the ambient environment (false positives) and version notice. + "--no-warn-conflicts", + "--disable-pip-version-check", + "--quiet", + ], + check=True, +) +shutil.copy(_SCRIPT_DIR / _AGENT_FILE, _PKG / _AGENT_FILE) + +_ZIP = _BUILD_DIR / "deployment_package.zip" +with zipfile.ZipFile(_ZIP, "w", zipfile.ZIP_DEFLATED) as zf: + for root, _, files in os.walk(_PKG): + for f in files: + if f.endswith(".pyc") or "__pycache__" in root: + continue + full = Path(root) / f + zf.write(full, full.relative_to(_PKG)) +print(f" Package: {_ZIP} ({_ZIP.stat().st_size / 1024 / 1024:.1f} MB)") + +# --------------------------------------------------------------------------- +# 3. Upload to S3 +# --------------------------------------------------------------------------- + +print("\n[4/6] Uploading to S3 ...") +try: + if REGION == "us-east-1": + _s3.create_bucket(Bucket=_S3_BUCKET) + else: + _s3.create_bucket( + Bucket=_S3_BUCKET, + CreateBucketConfiguration={"LocationConstraint": REGION}, + ) + print(f" Created bucket: {_S3_BUCKET}") +except Exception: + print(f" Bucket exists: {_S3_BUCKET}") +_s3.upload_file(str(_ZIP), _S3_BUCKET, _S3_KEY) +print(f" Uploaded: s3://{_S3_BUCKET}/{_S3_KEY}") + +# --------------------------------------------------------------------------- +# 4. Create AgentCore Runtime +# --------------------------------------------------------------------------- + +print(f"\n[5/6] Creating AgentCore Runtime '{_AGENT_NAME}' ...") +_resp = _ctrl.create_agent_runtime( + agentRuntimeName=_AGENT_NAME, + agentRuntimeArtifact={ + "codeConfiguration": { + "code": {"s3": {"bucket": _S3_BUCKET, "prefix": _S3_KEY}}, + "runtime": "PYTHON_3_13", + "entryPoint": ["opentelemetry-instrument", _AGENT_FILE], + } + }, + networkConfiguration={"networkMode": "PUBLIC"}, + roleArn=_ROLE_ARN, + environmentVariables={"AGENTCORE_MEMORY_ID": MEMORY_ID}, +) +AGENT_ID = _resp["agentRuntimeId"] +print(f" Runtime ID: {AGENT_ID}") + +# --------------------------------------------------------------------------- +# 5. Poll until READY +# --------------------------------------------------------------------------- + +print("\n[6/6] Waiting for READY ...") +for _elapsed in range(0, 600, 15): + _status = _ctrl.get_agent_runtime(agentRuntimeId=AGENT_ID).get("status", "UNKNOWN") + print(f" [{_elapsed:>3}s] {_status}") + if _status in ("READY", "ACTIVE"): + break + if "FAILED" in _status: + raise RuntimeError(f"Deploy failed: {_status}") + time.sleep(15) +else: + raise TimeoutError("Agent did not reach READY in 600s") + +AGENT_ARN = _ctrl.get_agent_runtime(agentRuntimeId=AGENT_ID)["agentRuntimeArn"] +CW_LOG_GROUP = f"/aws/bedrock-agentcore/runtimes/{AGENT_ID}-DEFAULT" + +# --------------------------------------------------------------------------- +# 6. Save agent_config.json +# --------------------------------------------------------------------------- + +_config = { + "agent_id": AGENT_ID, + "agent_arn": AGENT_ARN, + "cw_log_group": CW_LOG_GROUP, + "region": REGION, + "role_arn": _ROLE_ARN, + "s3_bucket": _S3_BUCKET, + "s3_key": _S3_KEY, + "memory_id": MEMORY_ID, +} +_CONFIG_FILE.write_text(json.dumps(_config, indent=2)) + +print("\nDeploy complete.") +print(f" AGENT_ID : {AGENT_ID}") +print(f" AGENT_ARN : {AGENT_ARN}") +print(f" CW_LOG_GROUP : {CW_LOG_GROUP}") +print(f" MEMORY_ID : {MEMORY_ID}") +print(f" Config saved : {_CONFIG_FILE}") diff --git a/01-features/06-observe-evaluate-optimize-your-agent/02-evaluate/supported-frameworks/openai-agents/evaluate.py b/01-features/06-observe-evaluate-optimize-your-agent/02-evaluate/supported-frameworks/openai-agents/evaluate.py new file mode 100644 index 00000000..44608d89 --- /dev/null +++ b/01-features/06-observe-evaluate-optimize-your-agent/02-evaluate/supported-frameworks/openai-agents/evaluate.py @@ -0,0 +1,598 @@ +""" +Evaluate the OpenAI Agents HR Assistant with AgentCore Evaluations. + +This is the same evaluation flow used by the framework-agnostic samples in +../../ — the driver only invokes the deployed runtime and calls the evaluation +APIs, so it is identical regardless of the agent framework. The agent behind +agent_config.json here is built with the OpenAI Agents SDK (see +openai_hr_assistant.py) and instrumented with the OpenTelemetry OpenAI Agents +library (scope opentelemetry.instrumentation.openai_agents). + +Two evaluation modes are demonstrated: + + 1. On-Demand Evaluation (EvaluationClient) + Invoke the agent for a session, then evaluate the recorded CloudWatch + spans immediately. Built-in evaluators + custom LLM-as-a-judge evaluators + run in the same call. Use for spot-checks and CI/CD regression tests. + + 2. Online Evaluation (create_online_evaluation_config) + Create a persistent config that continuously monitors the agent's live + traffic and scores every sampled session automatically. + +Usage: + python evaluate.py [--region REGION] [--config PATH] + +Args: + --region AWS region (default: from agent_config.json or boto3 session) + --config Path to agent_config.json written by deploy.py + (default: ./agent_config.json) + +Prerequisites: + 1. Deploy the OpenAI Agents HR Assistant: + python deploy.py [--region REGION] + 2. Install evaluation dependencies: + pip install -r requirements.txt + +Outputs: + results/on_demand_results.json - EvaluationClient scores + results/online_eval_config.json - Online evaluation config details +""" + +import argparse +import json +import sys +import time +import uuid +from pathlib import Path + +import boto3 +from boto3.session import Session + +# ============================================================ +# 0. Parse args and load agent config +# ============================================================ + +_SCRIPT_DIR = Path(__file__).parent +_DEFAULT_CONFIG = _SCRIPT_DIR / "agent_config.json" +_RESULTS_DIR = _SCRIPT_DIR / "results" +_RESULTS_DIR.mkdir(exist_ok=True) +_CLEANUP_STATE_PATH = _RESULTS_DIR / "cleanup_state.json" + + +def _load_cleanup_state() -> dict[str, object]: + """Load identifiers saved by previous complete or partial runs.""" + if not _CLEANUP_STATE_PATH.exists(): + return {} + state = json.loads(_CLEANUP_STATE_PATH.read_text()) + if not isinstance(state, dict): + raise ValueError(f"Expected a JSON object in {_CLEANUP_STATE_PATH}") + return state + + +_cleanup_state = _load_cleanup_state() + + +def _save_cleanup_state() -> None: + """Persist resource identifiers for cleanup after partial or complete runs.""" + _CLEANUP_STATE_PATH.write_text(json.dumps(_cleanup_state, indent=2)) + + +def _remember_cleanup_value(key: str, value: str) -> None: + """Append a resource identifier to cleanup state without duplicates.""" + existing = _cleanup_state.get(key) + values = [item for item in existing if isinstance(item, str)] if isinstance(existing, list) else [] + if value not in values: + values.append(value) + _cleanup_state[key] = values + _save_cleanup_state() + + +parser = argparse.ArgumentParser(description="Evaluate the OpenAI Agents HR Assistant") +parser.add_argument("--region", default=None, help="AWS region") +parser.add_argument( + "--config", + default=str(_DEFAULT_CONFIG), + help="Path to agent_config.json (written by deploy.py)", +) +args = parser.parse_args() + +_config_path = Path(args.config) +if not _config_path.exists(): + print(f"ERROR: Agent config not found at {_config_path}") + print("Run deploy.py first: python deploy.py") + sys.exit(1) + +_cfg = json.loads(_config_path.read_text()) +AGENT_ID = _cfg["agent_id"] +AGENT_ARN = _cfg["agent_arn"] +CW_LOG_GROUP = _cfg["cw_log_group"] +REGION = args.region or _cfg.get("region") or Session().region_name or "us-west-2" + +ACCOUNT_ID = boto3.client("sts", region_name=REGION).get_caller_identity()["Account"] + +# Derive OTel service name from agent ARN: +# ARN format: arn:aws:bedrock-agentcore:{region}:{account}:runtime/{id} +_runtime_id = AGENT_ARN.split("/")[-1] +_agent_runtime_name = _runtime_id.rsplit("-", 1)[0] +OTEL_SERVICE_NAME = f"{_agent_runtime_name}.DEFAULT" + +print("=" * 60) +print("OpenAI Agents HR Assistant — AgentCore Evaluation") +print("=" * 60) +print(f" Region : {REGION}") +print(f" Agent ID : {AGENT_ID}") +print(f" Agent ARN : {AGENT_ARN}") +print(f" CW Log Group : {CW_LOG_GROUP}") +print(f" OTel Service : {OTEL_SERVICE_NAME}") + +agentcore_client = boto3.client("bedrock-agentcore", region_name=REGION) +_cp = boto3.client("bedrock-agentcore-control", region_name=REGION) +iam_client = boto3.client("iam") + +# ============================================================ +# 1. Create custom LLM-as-a-judge evaluators +# ============================================================ +# +# Custom evaluators define quality criteria in natural language. +# The service substitutes ground-truth placeholders at evaluation time. +# +# Two evaluator types useful for HR assistants: +# - TRACE-level: score each agent response against the expected answer +# - SESSION-level: check whether the right tools were called and all +# assertions are satisfied across the whole conversation + +print("\n[1/4] Creating custom LLM-as-a-judge evaluators ...") + +_SUFFIX = uuid.uuid4().hex[:8] + +# ---- Trace-level: HR response quality -------------------------------- +print(" Creating HRResponseQuality (TRACE) ...") +_resp_quality = _cp.create_evaluator( + evaluatorName=f"HRResponseQuality_openai_{_SUFFIX}", + level="TRACE", + evaluatorConfig={ + "llmAsAJudge": { + "instructions": ( + "You are evaluating an HR assistant chatbot response.\n\n" + "Agent response: {assistant_turn}\n\n" + "Rate the quality of the agent's response on the following criteria:\n" + "1. ACCURACY: Facts, numbers, and dates are stated confidently and consistently\n" + "2. COMPLETENESS: The response fully addresses the user's request\n" + "3. PROFESSIONALISM: Tone is appropriate for an HR context\n\n" + "Assign a single overall quality rating." + ), + "ratingScale": { + "numerical": [ + { + "value": 0.0, + "label": "poor", + "definition": "Response is inaccurate, incomplete, or unprofessional.", + }, + { + "value": 0.5, + "label": "acceptable", + "definition": "Response is mostly correct but missing details or slightly off.", + }, + { + "value": 1.0, + "label": "excellent", + "definition": "Response is accurate, complete, and professionally written.", + }, + ] + }, + "modelConfig": { + "bedrockEvaluatorModelConfig": { + "modelId": "us.amazon.nova-pro-v1:0", + "inferenceConfig": {"maxTokens": 1024}, + } + }, + } + }, +) +CUSTOM_RESPONSE_QUALITY_ID = _resp_quality["evaluatorId"] +_remember_cleanup_value("custom_evaluator_ids", CUSTOM_RESPONSE_QUALITY_ID) +print(f" evaluatorId: {CUSTOM_RESPONSE_QUALITY_ID}") + +# ---- Session-level: HR session completeness -------------------------- +print(" Creating HRSessionCompleteness (SESSION) ...") +_session_check = _cp.create_evaluator( + evaluatorName=f"HRSessionCompleteness_openai_{_SUFFIX}", + level="SESSION", + evaluatorConfig={ + "llmAsAJudge": { + "instructions": ( + "You are reviewing a complete HR assistant conversation.\n\n" + "Expected tool trajectory: {expected_tool_trajectory}\n" + "Actual tool trajectory: {actual_tool_trajectory}\n" + "Session assertions: {assertions}\n\n" + "Evaluate whether the agent:\n" + "1. Called the expected tools (in any order)\n" + "2. Satisfied all session assertions\n" + "3. Reached a successful resolution for the user's request\n\n" + "Rate the overall session completeness." + ), + "ratingScale": { + "numerical": [ + { + "value": 0.0, + "label": "incomplete", + "definition": "Agent failed to call required tools or left the request unresolved.", + }, + { + "value": 0.5, + "label": "partial", + "definition": "Agent partially fulfilled the request — some tools missing or assertions unmet.", + }, + { + "value": 1.0, + "label": "complete", + "definition": "Agent called all expected tools and satisfied every assertion.", + }, + ] + }, + "modelConfig": { + "bedrockEvaluatorModelConfig": { + "modelId": "us.amazon.nova-pro-v1:0", + "inferenceConfig": {"maxTokens": 1024}, + } + }, + } + }, +) +CUSTOM_SESSION_COMPLETENESS_ID = _session_check["evaluatorId"] +_remember_cleanup_value("custom_evaluator_ids", CUSTOM_SESSION_COMPLETENESS_ID) +print(f" evaluatorId: {CUSTOM_SESSION_COMPLETENESS_ID}") + +# ============================================================ +# 2. Invoke agent to generate a session +# ============================================================ +# +# The OpenAI Agents HR assistant is already deployed (deploy.py). +# We invoke it for a multi-turn session so there are CloudWatch spans +# to evaluate. A unique runtimeSessionId groups all turns together. + +print("\n[2/4] Invoking HR Assistant to generate a session ...") + +SESSION_ID = f"openai-hr-eval-{uuid.uuid4()}" +print(f" Session ID: {SESSION_ID}") + +TURNS = [ + "What is the PTO balance for employee EMP-001?", + "Please submit a PTO request for EMP-001 from 2026-07-14 to 2026-07-18.", + "What is the company remote work policy?", +] + +EXPECTED_RESPONSES = [ + "Employee EMP-001 has 10 remaining PTO days out of 15 total (5 days used).", + "PTO request submitted for EMP-001 from 2026-07-14 to 2026-07-18. Request ID: PTO-2026-NNN.", + "The company allows up to 3 days of remote work per week. Core hours are 10am–3pm.", +] + +EXPECTED_TRAJECTORY = ["get_pto_balance", "submit_pto_request", "lookup_hr_policy"] + +ASSERTIONS = [ + "Agent called get_pto_balance with employee_id=EMP-001", + "Agent reported 10 remaining PTO days", + "Agent submitted a PTO request and returned a request ID", + "Agent described the remote work policy", +] + + +def _invoke_turn(prompt: str) -> str: + resp = agentcore_client.invoke_agent_runtime( + agentRuntimeArn=AGENT_ARN, + qualifier="DEFAULT", + runtimeSessionId=SESSION_ID, + payload=json.dumps({"prompt": prompt}).encode("utf-8"), + ) + raw = resp["response"].read().decode("utf-8") + parts = [] + for line in raw.splitlines(): + if line.startswith("data: "): + chunk = line[len("data: ") :] + try: + chunk = json.loads(chunk) + except Exception: + pass + parts.append(str(chunk)) + return "".join(parts) if parts else raw + + +for i, (prompt, expected) in enumerate(zip(TURNS, EXPECTED_RESPONSES), 1): + print(f" Turn {i}: {prompt[:70]}") + reply = _invoke_turn(prompt) + print(f" -> {reply[:100]}") + +print("\n Waiting 90s for CloudWatch span ingestion ...") +time.sleep(90) +print(" Ready for evaluation.") + +# ============================================================ +# 3. On-Demand Evaluation with EvaluationClient +# ============================================================ +# +# EvaluationClient evaluates the recorded session spans from CloudWatch. +# You can mix built-in evaluators with your custom LLM-as-a-judge evaluators +# in the same call. Provide ReferenceInputs ground truth to unlock evaluators +# that require expected responses or trajectories. + +from bedrock_agentcore.evaluation import EvaluationClient # noqa: E402 +from bedrock_agentcore.evaluation.client import ReferenceInputs # noqa: E402 +from datetime import timedelta # noqa: E402 + +print("\n[3/4] Running on-demand evaluation (EvaluationClient) ...") + +ec = EvaluationClient(region_name=REGION) + +# Pre-populate the evaluator level cache — required for Builtin.* evaluators +# because the SDK cannot resolve their level via GetEvaluator API. +ec._evaluator_level_cache.update( + { + "Builtin.GoalSuccessRate": "SESSION", + "Builtin.Correctness": "TRACE", + "Builtin.Helpfulness": "TRACE", + CUSTOM_RESPONSE_QUALITY_ID: "TRACE", + CUSTOM_SESSION_COMPLETENESS_ID: "SESSION", + } +) + +EVALUATOR_IDS = [ + "Builtin.GoalSuccessRate", # SESSION: did the agent meet the user's goal? + "Builtin.Correctness", # TRACE: is each response factually correct? + "Builtin.Helpfulness", # TRACE: was each response helpful? + CUSTOM_RESPONSE_QUALITY_ID, # TRACE: HR-specific response quality + CUSTOM_SESSION_COMPLETENESS_ID, # SESSION: did all assertions pass? +] + +# ReferenceInputs provide ground truth for evaluators that need it. +REFERENCE_INPUTS = ReferenceInputs( + assertions=ASSERTIONS, + expected_trajectory=EXPECTED_TRAJECTORY, + expected_response=EXPECTED_RESPONSES[-1], +) + +on_demand_results = ec.run( + evaluator_ids=EVALUATOR_IDS, + agent_id=AGENT_ID, + session_id=SESSION_ID, + look_back_time=timedelta(hours=1), + reference_inputs=REFERENCE_INPUTS, +) + +# Display results +print(f"\n Received {len(on_demand_results)} result(s):\n") +print(f" {'Evaluator':<45} {'Value':<8} {'Label'}") +print(" " + "-" * 80) + +for result in on_demand_results: + evaluator_id = result.get("evaluatorId", "") + name = ( + evaluator_id + if evaluator_id.startswith("Builtin.") + else ("HRResponseQuality" if evaluator_id == CUSTOM_RESPONSE_QUALITY_ID else "HRSessionCompleteness") + ) + value = result.get("value", result.get("score", "N/A")) + label = result.get("label", result.get("rating", "N/A")) + error = result.get("errorCode") + if error: + label = f"ERR:{error}" + print(f" {name:<45} {str(value):<8} {str(label)}") + +# Save results +_results_path = _RESULTS_DIR / "on_demand_results.json" +_results_path.write_text( + json.dumps( + { + "session_id": SESSION_ID, + "evaluators": EVALUATOR_IDS, + "custom_evaluator_ids": { + "HRResponseQuality": CUSTOM_RESPONSE_QUALITY_ID, + "HRSessionCompleteness": CUSTOM_SESSION_COMPLETENESS_ID, + }, + "results": on_demand_results, + }, + indent=2, + default=str, + ) +) +print(f"\n Results saved: {_results_path}") + +# ============================================================ +# 4. Online Evaluation Configuration +# ============================================================ +# +# Online evaluation monitors live agent traffic continuously. +# Create a config once; it evaluates every sampled session automatically. +# +# Note: Once a config is ENABLED, its evaluators are LOCKED. +# To update an evaluator: disable the config → update → re-enable. + +print("\n[4/4] Creating online evaluation configuration ...") + +# ---- 4a. IAM role for the evaluation service ------------------------- +ONLINE_EVAL_ROLE_NAME = f"AgentCoreOnlineEvalOpenAI_{_SUFFIX}" +ONLINE_EVAL_ROLE_ARN = f"arn:aws:iam::{ACCOUNT_ID}:role/{ONLINE_EVAL_ROLE_NAME}" + +_trust_policy = json.dumps( + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": {"Service": "bedrock-agentcore.amazonaws.com"}, + "Action": "sts:AssumeRole", + } + ], + } +) + +_inline_policy = json.dumps( + { + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "CloudWatchLogsReadWrite", + "Effect": "Allow", + "Action": [ + "logs:FilterLogEvents", + "logs:GetLogEvents", + "logs:DescribeLogGroups", + "logs:DescribeLogStreams", + "logs:StartQuery", + "logs:GetQueryResults", + "logs:StopQuery", + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:PutLogEvents", + ], + "Resource": "*", + }, + { + "Sid": "BedrockInvokeForJudge", + "Effect": "Allow", + "Action": [ + "bedrock:InvokeModel", + "bedrock:InvokeModelWithResponseStream", + ], + "Resource": "*", + }, + ], + } +) + +try: + iam_client.get_role(RoleName=ONLINE_EVAL_ROLE_NAME) + print(f" Using existing IAM role: {ONLINE_EVAL_ROLE_ARN}") +except iam_client.exceptions.NoSuchEntityException: + iam_client.create_role( + RoleName=ONLINE_EVAL_ROLE_NAME, + AssumeRolePolicyDocument=_trust_policy, + Description="Execution role for AgentCore online LLM-as-a-judge evaluation", + ) + print(f" Created IAM role: {ONLINE_EVAL_ROLE_ARN}") + +_remember_cleanup_value("evaluation_role_names", ONLINE_EVAL_ROLE_NAME) +iam_client.put_role_policy( + RoleName=ONLINE_EVAL_ROLE_NAME, + PolicyName="AgentCoreOnlineEvalPolicy", + PolicyDocument=_inline_policy, +) +print(" Waiting 10s for IAM propagation ...") +time.sleep(10) + +# ---- 4b. Create online evaluation config ---------------------------- +# Config name: alphanumeric + underscores only (no hyphens) +ONLINE_EVAL_CONFIG_NAME = f"hr_openai_eval_{_SUFFIX}" + +# Note: Custom evaluators that use reference input placeholders +# ({expected_response}, {assertions}, etc.) require ground truth and therefore +# can only be used in on-demand evaluation. Online evaluation evaluates live +# traffic where no ground truth is available, so only built-in evaluators +# (or custom evaluators without reference inputs) are supported here. +_ONLINE_EVALUATORS = [ + "Builtin.GoalSuccessRate", + "Builtin.Correctness", + "Builtin.Helpfulness", +] + +print(f" Config name : {ONLINE_EVAL_CONFIG_NAME}") +print(f" Log group : {CW_LOG_GROUP}") +print(f" OTel service : {OTEL_SERVICE_NAME}") +print(f" Evaluators : {', '.join(_ONLINE_EVALUATORS)}") +print(" Note: Custom evaluators with reference inputs are on-demand only") + +_online_resp = _cp.create_online_evaluation_config( + onlineEvaluationConfigName=ONLINE_EVAL_CONFIG_NAME, + # 100% sampling in this example; lower for high-traffic production agents + rule={"samplingConfig": {"samplingPercentage": 100.0}}, + dataSourceConfig={ + "cloudWatchLogs": { + "logGroupNames": [CW_LOG_GROUP], + "serviceNames": [OTEL_SERVICE_NAME], + } + }, + evaluators=[{"evaluatorId": eid} for eid in _ONLINE_EVALUATORS], + evaluationExecutionRoleArn=ONLINE_EVAL_ROLE_ARN, + enableOnCreate=True, +) + +ONLINE_CONFIG_ID = _online_resp["onlineEvaluationConfigId"] +ONLINE_CONFIG_ARN = _online_resp.get("onlineEvaluationConfigArn", "") +_remember_cleanup_value("online_evaluation_config_ids", ONLINE_CONFIG_ID) +_remember_cleanup_value( + "results_log_groups", + f"/aws/bedrock-agentcore/evaluations/results/{ONLINE_CONFIG_ID}", +) + +print("\n Online evaluation config created:") +print(f" ID : {ONLINE_CONFIG_ID}") +print(f" ARN : {ONLINE_CONFIG_ARN}") +print() +print(" The config is now ACTIVE. Every new HR assistant session will be") +print(" automatically evaluated with built-in evaluators.") +print(" Results appear in CloudWatch at:") +print(f" /aws/bedrock-agentcore/evaluations/results/{ONLINE_CONFIG_ID}") + +# ---- 4c. Invoke agent to trigger online evaluation ------------------ +print("\n Invoking agent to trigger a live online evaluation ...") + +_online_session = f"online-openai-{uuid.uuid4()}" +_online_prompts = [ + "What is the PTO balance for employee EMP-042?", + "What health insurance options does the company offer?", +] + +for prompt in _online_prompts: + print(f" > {prompt[:70]}") + reply = agentcore_client.invoke_agent_runtime( + agentRuntimeArn=AGENT_ARN, + qualifier="DEFAULT", + runtimeSessionId=_online_session, + payload=json.dumps({"prompt": prompt}).encode("utf-8"), + ) + reply.get("response", b"").read() # consume stream + +print(" Online evaluation will score this session automatically.") +print(" Results appear in CloudWatch within a few minutes.") + +# Save online eval config details +_online_path = _RESULTS_DIR / "online_eval_config.json" +_online_path.write_text( + json.dumps( + { + "config_name": ONLINE_EVAL_CONFIG_NAME, + "config_id": ONLINE_CONFIG_ID, + "config_arn": ONLINE_CONFIG_ARN, + "custom_evaluator_ids": { + "HRResponseQuality": CUSTOM_RESPONSE_QUALITY_ID, + "HRSessionCompleteness": CUSTOM_SESSION_COMPLETENESS_ID, + }, + "evaluation_role_name": ONLINE_EVAL_ROLE_NAME, + "evaluation_role_arn": ONLINE_EVAL_ROLE_ARN, + "triggered_session_id": _online_session, + "results_log_group": f"/aws/bedrock-agentcore/evaluations/results/{ONLINE_CONFIG_ID}", + }, + indent=2, + ) +) +print(f"\n Config details saved: {_online_path}") + +# ============================================================ +# Summary +# ============================================================ + +print("\n" + "=" * 60) +print("Summary") +print("=" * 60) +print(" Custom evaluators created : HRResponseQuality, HRSessionCompleteness") +print(f" On-demand evaluation : {len(on_demand_results)} result(s) for session {SESSION_ID[:20]}...") +print(f" Online eval config : {ONLINE_EVAL_CONFIG_NAME} (ENABLED)") +print() +print(" Next steps:") +print(" - Check on-demand scores: results/on_demand_results.json") +print(" - Monitor online eval: AWS Console → CloudWatch → Log groups") +print(f" /aws/bedrock-agentcore/evaluations/results/{ONLINE_CONFIG_ID}") +print(" - Disable online config when done:") +print(" aws bedrock-agentcore-control update-online-evaluation-config \\") +print(f" --online-evaluation-config-id {ONLINE_CONFIG_ID} \\") +print(" --execution-status DISABLED") diff --git a/01-features/06-observe-evaluate-optimize-your-agent/02-evaluate/supported-frameworks/openai-agents/images/architecture.png b/01-features/06-observe-evaluate-optimize-your-agent/02-evaluate/supported-frameworks/openai-agents/images/architecture.png new file mode 100644 index 00000000..f2775d79 Binary files /dev/null and b/01-features/06-observe-evaluate-optimize-your-agent/02-evaluate/supported-frameworks/openai-agents/images/architecture.png differ diff --git a/01-features/06-observe-evaluate-optimize-your-agent/02-evaluate/supported-frameworks/openai-agents/images/sample-trace.png b/01-features/06-observe-evaluate-optimize-your-agent/02-evaluate/supported-frameworks/openai-agents/images/sample-trace.png new file mode 100644 index 00000000..8a9af744 Binary files /dev/null and b/01-features/06-observe-evaluate-optimize-your-agent/02-evaluate/supported-frameworks/openai-agents/images/sample-trace.png differ diff --git a/01-features/06-observe-evaluate-optimize-your-agent/02-evaluate/supported-frameworks/openai-agents/openai_hr_assistant.py b/01-features/06-observe-evaluate-optimize-your-agent/02-evaluate/supported-frameworks/openai-agents/openai_hr_assistant.py new file mode 100644 index 00000000..512c3656 --- /dev/null +++ b/01-features/06-observe-evaluate-optimize-your-agent/02-evaluate/supported-frameworks/openai-agents/openai_hr_assistant.py @@ -0,0 +1,433 @@ +""" +HR Assistant Agent: OpenAI Agents SDK agent deployed on Bedrock AgentCore Runtime. + +Same HR Assistant domain as the shared Strands agent in ../../utils, re-implemented +with the OpenAI Agents SDK so it can be evaluated with AgentCore Evaluations. The +tools, mock data, and system prompt are identical, so ground-truth and expected +responses stay consistent across framework samples. + +The LLM is OpenAI GPT-5.5 on Amazon Bedrock, reached through the Bedrock mantle +endpoint's OpenAI-compatible Responses API and authenticated with a Bedrock API +key. aws_bedrock_token_generator.provide_token() mints a short-term Bedrock API +key from the runtime's IAM role on every invocation — the secure, recommended +kind, so no key is stored in code or config. + +The Responses API (OpenAIResponsesModel) is used rather than Chat Completions: +the OpenTelemetry instrumentation extracts the agent's response text from +Responses API spans (ResponseSpanData), which AgentCore Evaluations needs to +score the agent's answers. + +Conversation history is persisted in AgentCore Memory (short-term memory +events) per runtime session, so multi-turn context survives microVM restarts. +deploy.py creates the memory resource and injects AGENTCORE_MEMORY_ID. + +Observability is provided by ADOT with the OpenTelemetry OpenAI Agents +instrumentation (added to requirements.txt). ADOT discovers it at startup, so no +explicit instrumentation code is needed here. The instrumentation hooks into the +SDK's tracing pipeline, so SDK tracing must stay enabled (do NOT call +set_tracing_disabled) — the SDK's default platform.openai.com exporter is inert +without an OPENAI_API_KEY and only logs a skip message. + +Tools (deterministic / mock data for reproducible evaluations): + get_pto_balance - remaining PTO days for an employee + submit_pto_request - request time off + lookup_hr_policy - company policy documents + get_benefits_summary - health, dental, vision, 401k, life insurance details + get_pay_stub - pay stub for a given period +""" + +import logging +import os +import re + +from agents import ( + Agent, + OpenAIResponsesModel, + Runner, + function_tool, +) +from aws_bedrock_token_generator import provide_token +from bedrock_agentcore.memory import MemoryClient +from bedrock_agentcore.runtime import BedrockAgentCoreApp +from openai import AsyncOpenAI + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +app = BedrockAgentCoreApp() + +# --------------------------------------------------------------------------- +# Model configuration (OpenAI GPT-5.5 on Bedrock via the mantle endpoint) +# --------------------------------------------------------------------------- +# +# GPT-5.5 is served on the mantle endpoint's openai/v1 path (a different path +# from the /v1 used by gpt-oss). It is available in us-east-1 / us-east-2, so +# MODEL_REGION may differ from the region the runtime is deployed in. The +# runtime role needs bedrock-mantle:CreateInference and +# bedrock-mantle:CallWithBearerToken (granted by deploy.py). + +REGION = os.environ.get("AWS_REGION", "us-west-2") +MODEL_REGION = os.environ.get("BEDROCK_OPENAI_MODEL_REGION", "us-east-1") +BASE_URL = os.environ.get("BEDROCK_OPENAI_BASE_URL", f"https://bedrock-mantle.{MODEL_REGION}.api.aws/openai/v1") +MODEL_ID = os.environ.get("BEDROCK_OPENAI_MODEL_ID", "openai.gpt-5.5") + +# AgentCore Memory holds the conversation history across turns (and across +# microVM restarts). The memory resource is created by deploy.py and its id is +# injected as an environment variable on the runtime. +MEMORY_ID = os.environ.get("AGENTCORE_MEMORY_ID", "") +ACTOR_ID = "hr-employee" +_memory_client = MemoryClient(region_name=REGION) if MEMORY_ID else None + +# NOTE: Do not call set_tracing_disabled(True) here. The OpenTelemetry +# instrumentation registers a processor on the SDK's tracing pipeline; disabling +# SDK tracing would silence the evaluation spans. The SDK's default +# platform.openai.com exporter skips exporting when OPENAI_API_KEY is unset. + +# --------------------------------------------------------------------------- +# Mock data +# --------------------------------------------------------------------------- + +_PTO_BALANCES = { + "EMP-001": {"total_days": 15, "used_days": 5, "remaining_days": 10}, + "EMP-002": {"total_days": 15, "used_days": 12, "remaining_days": 3}, + "EMP-042": {"total_days": 20, "used_days": 7, "remaining_days": 13}, +} + +_HR_POLICIES = { + "pto": ( + "PTO Policy: Full-time employees accrue 15 days of PTO per year (20 days after 3 years). " + "PTO requests must be submitted at least 2 business days in advance. " + "Unused PTO up to 5 days rolls over to the next year. " + "PTO cannot be taken in advance of accrual." + ), + "remote_work": ( + "Remote Work Policy: Employees may work remotely up to 3 days per week with manager approval. " + "Core collaboration hours are 10am-3pm local time. " + "A dedicated workspace with reliable internet (25 Mbps+) is required. " + "Employees must be reachable via Slack and email during core hours." + ), + "parental_leave": ( + "Parental Leave Policy: Primary caregivers receive 16 weeks of fully paid parental leave. " + "Secondary caregivers receive 6 weeks of fully paid parental leave. " + "Leave may begin up to 2 weeks before the expected birth or adoption date. " + "Benefits continue unchanged during parental leave." + ), + "code_of_conduct": ( + "Code of Conduct: All employees are expected to treat colleagues, customers, and partners " + "with respect and professionalism. Harassment, discrimination, and retaliation of any kind " + "are strictly prohibited. Violations should be reported to HR or via the anonymous hotline." + ), +} + +_BENEFITS = { + "health": ( + "Health Insurance: The company covers 90% of premiums for employee-only coverage and 75% " + "for family coverage. Plans available: Blue Shield PPO, Kaiser HMO, and HDHP with HSA. " + "Annual deductible: $500 (PPO), $0 (HMO), $1,500 (HDHP). " + "Open enrollment is each November for the following calendar year." + ), + "dental": ( + "Dental Insurance: 100% coverage for preventive care (cleanings, X-rays). " + "80% coverage for basic restorative care (fillings, extractions). " + "50% coverage for major restorative care (crowns, bridges). " + "Annual maximum benefit: $2,000 per person. Orthodontia lifetime maximum: $1,500." + ), + "vision": ( + "Vision Insurance: Annual eye exam covered in full. " + "Frames or contacts allowance: $200 per year. " + "Laser vision correction discount: 15% off at participating providers." + ), + "401k": ( + "401(k) Plan: The company matches 100% of employee contributions up to 4% of salary. " + "An additional 50% match on the next 2% (total effective match up to 5%). " + "Employees are eligible to contribute immediately; company match vests over 3 years. " + "2026 IRS contribution limit: $23,500 (under 50), $31,000 (age 50+)." + ), + "life_insurance": ( + "Life Insurance: Basic life insurance of 2x annual salary provided at no cost. " + "Employees may purchase supplemental coverage up to 5x salary during open enrollment. " + "Accidental death and dismemberment (AD&D) coverage equal to basic life benefit is included." + ), +} + +_PAY_STUBS = { + ("EMP-001", "2025-12"): { + "gross_pay": 8333.33, + "federal_tax": 1458.33, + "state_tax": 416.67, + "social_security": 516.67, + "medicare": 120.83, + "health_premium": 125.00, + "401k_contribution": 333.33, + "net_pay": 5362.50, + "period": "December 2025", + }, + ("EMP-001", "2026-01"): { + "gross_pay": 8333.33, + "federal_tax": 1458.33, + "state_tax": 416.67, + "social_security": 516.67, + "medicare": 120.83, + "health_premium": 125.00, + "401k_contribution": 333.33, + "net_pay": 5362.50, + "period": "January 2026", + }, + ("EMP-042", "2026-01"): { + "gross_pay": 10416.67, + "federal_tax": 1875.00, + "state_tax": 520.83, + "social_security": 645.83, + "medicare": 151.04, + "health_premium": 200.00, + "401k_contribution": 416.67, + "net_pay": 6607.30, + "period": "January 2026", + }, +} + +_PTO_REQUEST_COUNTER = {"n": 0} + + +# --------------------------------------------------------------------------- +# OpenAI Agents SDK tools +# --------------------------------------------------------------------------- + + +@function_tool +def get_pto_balance(employee_id: str) -> dict: + """ + Return the current PTO balance for an employee. + + Args: + employee_id: Employee identifier (e.g. EMP-001) + + Returns: + Dict with total_days, used_days, and remaining_days. + """ + balance = _PTO_BALANCES.get(employee_id) + if balance: + return {"employee_id": employee_id, **balance} + return {"employee_id": employee_id, "error": f"Employee {employee_id} not found."} + + +@function_tool +def submit_pto_request( + employee_id: str, + start_date: str, + end_date: str, + reason: str = "Personal time off", +) -> dict: + """ + Submit a PTO request for an employee. + + Args: + employee_id: Employee identifier (e.g. EMP-001) + start_date: First day of leave in YYYY-MM-DD format + end_date: Last day of leave in YYYY-MM-DD format + reason: Optional reason for the request + + Returns: + Dict with request_id, status, and confirmation message. + """ + _PTO_REQUEST_COUNTER["n"] += 1 + request_id = f"PTO-2026-{_PTO_REQUEST_COUNTER['n']:03d}" + return { + "request_id": request_id, + "employee_id": employee_id, + "start_date": start_date, + "end_date": end_date, + "reason": reason, + "status": "APPROVED", + "message": f"PTO request {request_id} approved for {employee_id} from {start_date} to {end_date}.", + } + + +@function_tool +def lookup_hr_policy(topic: str) -> dict: + """ + Look up a company HR policy document by topic. + + Args: + topic: Policy topic. Supported values: pto, remote_work, parental_leave, code_of_conduct + + Returns: + Dict with topic and policy_text. + """ + key = topic.lower().replace(" ", "_").replace("-", "_") + text = _HR_POLICIES.get(key) + if text: + return {"topic": topic, "policy_text": text} + return { + "topic": topic, + "error": f"Policy '{topic}' not found. Available: {list(_HR_POLICIES.keys())}", + } + + +@function_tool +def get_benefits_summary(benefit_type: str) -> dict: + """ + Return a summary of a specific employee benefit. + + Args: + benefit_type: Type of benefit. Supported values: health, dental, vision, 401k, life_insurance + + Returns: + Dict with benefit_type and summary text. + """ + key = benefit_type.lower().replace(" ", "_").replace("-", "_") + text = _BENEFITS.get(key) + if text: + return {"benefit_type": benefit_type, "summary": text} + return { + "benefit_type": benefit_type, + "error": f"Benefit '{benefit_type}' not found. Available: {list(_BENEFITS.keys())}", + } + + +@function_tool +def get_pay_stub(employee_id: str, period: str) -> dict: + """ + Retrieve a pay stub for an employee for a specific pay period. + + Args: + employee_id: Employee identifier (e.g. EMP-001) + period: Pay period in YYYY-MM format (e.g. 2026-01) + + Returns: + Dict with gross pay, deductions, and net pay. + """ + stub = _PAY_STUBS.get((employee_id, period)) + if stub: + return {"employee_id": employee_id, **stub} + return { + "employee_id": employee_id, + "period": period, + "error": f"Pay stub not found for {employee_id} period {period}.", + } + + +# --------------------------------------------------------------------------- +# Agent +# --------------------------------------------------------------------------- + +SYSTEM_PROMPT = """You are a helpful HR Assistant for Acme Corp. + +You help employees with: +- Checking PTO (paid time off) balances +- Submitting PTO requests +- Looking up HR policies (PTO, remote work, parental leave, code of conduct) +- Understanding employee benefits (health, dental, vision, 401k, life insurance) +- Retrieving pay stub information + +Always use the available tools to answer questions accurately. Do not make up +policy details, benefit amounts, or pay information. Look them up. +Be concise, professional, and friendly.""" + +_TOOLS = [ + get_pto_balance, + submit_pto_request, + lookup_hr_policy, + get_benefits_summary, + get_pay_stub, +] + + +def _build_agent() -> Agent: + """ + Build the HR Assistant agent. + + provide_token() returns a short-term Bedrock API key (a bedrock-api-key-... + string) minted from the runtime's IAM role credentials — a local SigV4 + presign with no network call. The agent is rebuilt on every invocation + rather than cached for the microVM's lifetime, so a long-lived runtime never + keeps using an expired key. + """ + api_key = provide_token(region=MODEL_REGION) + client = AsyncOpenAI(base_url=BASE_URL, api_key=api_key) + model = OpenAIResponsesModel(model=MODEL_ID, openai_client=client) + return Agent(name="HRAssistant", instructions=SYSTEM_PROMPT, model=model, tools=_TOOLS) + + +# Conversation history lives in AgentCore Memory (short-term memory events), +# keyed by the runtime session id. It survives microVM restarts and is shared +# with the AgentCore Memory console/APIs. +# +# The SDK's SQLiteSession is not used here for two reasons: it is local to one +# microVM (history is lost when the runtime scales or restarts), and it replays +# full Responses API output items (including model "reasoning" items) as the +# next turn's input, which the Bedrock mantle endpoint rejects with an empty +# output. Plain role/content text history from Memory round-trips reliably. + + +def _load_history(session_id: str) -> list: + """Load the conversation as [{"role", "content"}] items from AgentCore Memory.""" + if not _memory_client: + return [] + history = [] + events = _memory_client.list_events(memory_id=MEMORY_ID, actor_id=ACTOR_ID, session_id=session_id) + for event in sorted(events, key=lambda e: e["eventId"]): + for item in event.get("payload", []): + conv = item.get("conversational") + if conv: + role = "user" if conv["role"] == "USER" else "assistant" + history.append({"role": role, "content": conv["content"]["text"]}) + return history + + +def _save_turn(session_id: str, prompt: str, response: str): + """Persist one user/assistant turn to AgentCore Memory.""" + if not _memory_client: + return + _memory_client.create_event( + memory_id=MEMORY_ID, + actor_id=ACTOR_ID, + session_id=session_id, + messages=[(prompt, "USER"), (response, "ASSISTANT")], + ) + + +def _flush_telemetry(): + """ + Flush buffered OTel spans and event records before the microVM freezes. + + AgentCore Runtime suspends the microVM between invocations. Without an + explicit flush, event records buffered in the OTel batch processors (which + carry the agent's response text for evaluation) can be lost, and evaluators + then score empty responses. + """ + try: + from opentelemetry import trace as _trace + from opentelemetry._logs import get_logger_provider as _get_lp + + for provider in (_trace.get_tracer_provider(), _get_lp()): + flush = getattr(provider, "force_flush", None) + if flush: + flush() + except Exception: + logger.warning("Telemetry flush failed", exc_info=True) + + +@app.entrypoint +async def invoke(payload, context): + """Handle an agent invocation from AgentCore Runtime.""" + prompt = payload.get("prompt", "") + session_id = context.session_id or "default" + logger.info("Received prompt (session=%s): %s", session_id, prompt[:80]) + + history = _load_history(session_id) + history.append({"role": "user", "content": prompt}) + try: + result = await Runner.run(_build_agent(), history) + finally: + _flush_telemetry() + response = str(result.final_output) + # Some OpenAI models (e.g. gpt-oss) emit inline ... + # blocks; strip them so spans contain only the final answer + response = re.sub(r".*?", "", response, flags=re.DOTALL).strip() + _save_turn(session_id, prompt, response) + return response + + +if __name__ == "__main__": + app.run() diff --git a/01-features/06-observe-evaluate-optimize-your-agent/02-evaluate/supported-frameworks/openai-agents/requirements.txt b/01-features/06-observe-evaluate-optimize-your-agent/02-evaluate/supported-frameworks/openai-agents/requirements.txt new file mode 100644 index 00000000..d8fee7b9 --- /dev/null +++ b/01-features/06-observe-evaluate-optimize-your-agent/02-evaluate/supported-frameworks/openai-agents/requirements.txt @@ -0,0 +1,2 @@ +bedrock-agentcore>=1.6.0 +boto3>=1.43.0 diff --git a/01-features/06-observe-evaluate-optimize-your-agent/02-evaluate/supported-frameworks/tests/test_cleanup.py b/01-features/06-observe-evaluate-optimize-your-agent/02-evaluate/supported-frameworks/tests/test_cleanup.py new file mode 100644 index 00000000..b0d7021c --- /dev/null +++ b/01-features/06-observe-evaluate-optimize-your-agent/02-evaluate/supported-frameworks/tests/test_cleanup.py @@ -0,0 +1,210 @@ +"""Unit tests for the framework cleanup scripts.""" + +from __future__ import annotations + +import importlib.util +import json +from collections.abc import Iterator +from pathlib import Path +from types import ModuleType +from typing import Any, cast + +import pytest +from botocore.exceptions import ClientError + +_SUPPORTED_FRAMEWORKS_DIR = Path(__file__).parents[1] +_FRAMEWORKS = ("openai-agents", "llamaindex") + + +def _load_cleanup_module(framework: str) -> ModuleType: + path = _SUPPORTED_FRAMEWORKS_DIR / framework / "cleanup.py" + spec = importlib.util.spec_from_file_location(f"{framework}_cleanup", path) + if spec is None or spec.loader is None: + raise RuntimeError(f"Unable to load {path}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@pytest.fixture(params=_FRAMEWORKS) +def cleanup_module(request: pytest.FixtureRequest) -> ModuleType: + """Load each framework cleanup script as a module.""" + return _load_cleanup_module(cast(str, request.param)) + + +def _client_error(code: str) -> ClientError: + return ClientError( + { + "Error": {"Code": code, "Message": "test error"}, + "ResponseMetadata": { + "RequestId": "test-request", + "HostId": "", + "HTTPStatusCode": 400, + "HTTPHeaders": {}, + "RetryAttempts": 0, + }, + }, + "TestOperation", + ) + + +def test_collect_evaluation_state_merges_old_and_new_formats( + cleanup_module: ModuleType, + tmp_path: Path, +) -> None: + (tmp_path / "on_demand_results.json").write_text( + json.dumps( + { + "custom_evaluator_ids": { + "builtin": "Builtin.Correctness", + "custom": "custom-old", + } + } + ) + ) + (tmp_path / "online_eval_config.json").write_text( + json.dumps( + { + "online_evaluation_config_id": "config-old", + "evaluation_role_name": "role-old", + "results_log_group": "/aws/results/old", + } + ) + ) + (tmp_path / "cleanup_state.json").write_text( + json.dumps( + { + "online_evaluation_config_ids": ["config-old", "config-new"], + "custom_evaluator_ids": ["custom-old", "custom-new"], + "evaluation_role_names": ["role-old", "role-new"], + "results_log_groups": ["/aws/results/old", "/aws/results/new"], + } + ) + ) + + state = cleanup_module._collect_evaluation_state(tmp_path) + + assert state == { + "online_config_ids": ["config-new", "config-old"], + "custom_evaluator_ids": ["custom-new", "custom-old"], + "evaluation_role_names": ["role-new", "role-old"], + "results_log_groups": ["/aws/results/new", "/aws/results/old"], + } + + +def test_async_delete_waits_until_resource_is_absent( + cleanup_module: ModuleType, + monkeypatch: pytest.MonkeyPatch, +) -> None: + responses: Iterator[dict[str, str] | ClientError] = iter( + [ + {"status": "ACTIVE"}, + {"status": "DELETING"}, + _client_error("ResourceNotFoundException"), + ] + ) + delete_calls: list[bool] = [] + + def get_action() -> dict[str, str]: + response = next(responses) + if isinstance(response, ClientError): + raise response + return response + + def delete_action() -> None: + delete_calls.append(True) + + monkeypatch.setattr(cleanup_module.time, "sleep", lambda _: None) + failures: list[str] = [] + + deleted = cleanup_module._delete_async_resource( + "test resource", + delete_action, + get_action, + cleanup_module._flat_status, + failures, + poll_interval=0, + timeout=1, + ) + + assert deleted is True + assert delete_calls == [True] + assert failures == [] + + +def test_async_delete_is_idempotent_when_resource_is_absent(cleanup_module: ModuleType) -> None: + def get_action() -> dict[str, str]: + raise _client_error("ResourceNotFoundException") + + def delete_action() -> None: + raise AssertionError("delete must not be called") + + failures: list[str] = [] + deleted = cleanup_module._delete_async_resource( + "missing resource", + delete_action, + get_action, + cleanup_module._flat_status, + failures, + poll_interval=0, + timeout=0, + ) + + assert deleted is True + assert failures == [] + + +class _Paginator: + def __init__(self, pages: list[dict[str, Any]]) -> None: + self._pages = pages + + def paginate(self, **_: str) -> list[dict[str, Any]]: + return self._pages + + +class _IamClient: + def __init__(self) -> None: + self.calls: list[tuple[str, str]] = [] + + def get_paginator(self, name: str) -> _Paginator: + if name == "list_role_policies": + return _Paginator([{"PolicyNames": ["inline-a", "inline-b"]}]) + return _Paginator([{"AttachedPolicies": [{"PolicyArn": "arn:aws:iam::aws:policy/ReadOnlyAccess"}]}]) + + def delete_role_policy(self, *, RoleName: str, PolicyName: str) -> None: + self.calls.append(("delete-inline", PolicyName)) + + def detach_role_policy(self, *, RoleName: str, PolicyArn: str) -> None: + self.calls.append(("detach", PolicyArn)) + + def delete_role(self, *, RoleName: str) -> None: + self.calls.append(("delete-role", RoleName)) + + +def test_delete_iam_role_removes_policies_first(cleanup_module: ModuleType) -> None: + iam = _IamClient() + + cleanup_module._delete_iam_role(iam, "sample-role") + + assert iam.calls == [ + ("delete-inline", "inline-a"), + ("delete-inline", "inline-b"), + ("detach", "arn:aws:iam::aws:policy/ReadOnlyAccess"), + ("delete-role", "sample-role"), + ] + + +def test_main_fails_before_creating_session_when_config_is_missing( + cleanup_module: ModuleType, + tmp_path: Path, +) -> None: + exit_code = cleanup_module.main( + [ + "--config", + str(tmp_path / "missing.json"), + "--results-dir", + str(tmp_path), + ] + ) + + assert exit_code == 1 diff --git a/01-features/06-observe-evaluate-optimize-your-agent/README.md b/01-features/06-observe-evaluate-optimize-your-agent/README.md index c02ccbe6..24f572b6 100644 --- a/01-features/06-observe-evaluate-optimize-your-agent/README.md +++ b/01-features/06-observe-evaluate-optimize-your-agent/README.md @@ -6,7 +6,7 @@ automated evaluation, and AI-driven optimization with A/B testing. ## Overview -![Agent development to production loops](AGENT-LOOPS.png) +![Agent development to production loops](AGENT-LOOPS.PNG) ## Before You Start — Enable observability Infrastructure