{"id":50396210,"url":"https://awslabs.github.io/agent-evaluation/","last_synced_at":"2026-06-16T13:00:39.089Z","repository":{"id":237547762,"uuid":"774582281","full_name":"awslabs/agent-evaluation","owner":"awslabs","description":"A generative AI-powered framework for testing virtual agents.","archived":false,"fork":false,"pushed_at":"2025-12-15T18:10:34.000Z","size":187031,"stargazers_count":358,"open_issues_count":23,"forks_count":48,"subscribers_count":3,"default_branch":"main","last_synced_at":"2026-04-17T05:07:24.732Z","etag":null,"topics":[],"latest_commit_sha":null,"homepage":"https://awslabs.github.io/agent-evaluation/","language":"Python","has_issues":true,"has_wiki":null,"has_pages":null,"mirror_url":null,"source_name":null,"license":"apache-2.0","status":null,"scm":"git","pull_requests_enabled":true,"icon_url":"https://github.com/awslabs.png","metadata":{"files":{"readme":"README.md","changelog":"CHANGELOG.md","contributing":"CONTRIBUTING.md","funding":null,"license":"LICENSE","code_of_conduct":"CODE_OF_CONDUCT.md","threat_model":null,"audit":null,"citation":null,"codeowners":null,"security":null,"support":null,"governance":null,"roadmap":null,"authors":null,"dei":null,"publiccode":null,"codemeta":null,"zenodo":null,"notice":"NOTICE","maintainers":null,"copyright":null,"agents":null,"dco":null,"cla":null}},"created_at":"2024-03-19T19:58:26.000Z","updated_at":"2026-04-17T02:22:53.000Z","dependencies_parsed_at":"2024-05-30T02:09:44.373Z","dependency_job_id":"1e4b0b1b-c8bc-4210-b472-8c135cf0fe70","html_url":"https://github.com/awslabs/agent-evaluation","commit_stats":null,"previous_names":["awslabs/agent-evaluation"],"tags_count":5,"template":false,"template_full_name":"amazon-archives/__template_Apache-2.0","purl":"pkg:github/awslabs/agent-evaluation","repository_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/awslabs%2Fagent-evaluation","tags_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/awslabs%2Fagent-evaluation/tags","releases_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/awslabs%2Fagent-evaluation/releases","manifests_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/awslabs%2Fagent-evaluation/manifests","owner_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners/awslabs","download_url":"https://codeload.github.com/awslabs/agent-evaluation/tar.gz/refs/heads/main","sbom_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/awslabs%2Fagent-evaluation/sbom","scorecard":null,"host":{"name":"GitHub","url":"https://github.com","kind":"github","repositories_count":286080680,"owners_count":34406824,"icon_url":"https://github.com/github.png","version":null,"created_at":"2022-05-30T11:31:42.601Z","updated_at":"2026-05-26T15:22:16.424Z","status":"online","status_checked_at":"2026-06-16T02:00:06.860Z","response_time":126,"last_error":null,"robots_txt_status":"success","robots_txt_updated_at":"2025-07-24T06:49:26.215Z","robots_txt_url":"https://github.com/robots.txt","online":true,"can_crawl_api":true,"host_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub","repositories_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories","repository_names_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repository_names","owners_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners"}},"keywords":[],"created_at":"2026-05-30T21:01:21.423Z","updated_at":"2026-06-16T13:00:39.084Z","avatar_url":"https://github.com/awslabs.png","language":"Python","funding_links":[],"categories":["Browse The Shelves"],"sub_categories":["Agent evals"],"readme":"![PyPI - Version](https://img.shields.io/pypi/v/agent-evaluation)\n![PyPI - Python Version](https://img.shields.io/pypi/pyversions/agent-evaluation)\n![GitHub License](https://img.shields.io/github/license/awslabs/agent-evaluation)\n[![security: bandit](https://img.shields.io/badge/security-bandit-yellow.svg)](https://github.com/PyCQA/bandit)\n[![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)\n[![Built with Material for MkDocs](https://img.shields.io/badge/Material_for_MkDocs-526CFE?style=for-the-badge\u0026logo=MaterialForMkDocs\u0026logoColor=white)](https://squidfunk.github.io/mkdocs-material/)\n\n# Agent Evaluation\n\nAgent Evaluation is a generative AI-powered framework for testing virtual agents.\n\nInternally, Agent Evaluation implements an LLM agent (evaluator) that will orchestrate conversations with your own agent (target) and evaluate the responses during the conversation.\n\n## ✨ Key features\n\n- Built-in support for popular AWS services including [Amazon Bedrock](https://aws.amazon.com/bedrock/), [Amazon Q Business](https://aws.amazon.com/q/business/), and [Amazon SageMaker](https://aws.amazon.com/sagemaker/). You can also [bring your own agent](https://awslabs.github.io/agent-evaluation/targets/custom_targets/) to test using Agent Evaluation.\n- Orchestrate concurrent, multi-turn conversations with your agent while evaluating its responses.\n- Define [hooks](https://awslabs.github.io/agent-evaluation/hooks/) to perform additional tasks such as integration testing.\n- Can be incorporated into CI/CD pipelines to expedite the time to delivery while maintaining the stability of agents in production environments.\n\n## 📚 Documentation\n\nTo get started, please visit the full documentation [here](https://awslabs.github.io/agent-evaluation/). To contribute, please refer to [CONTRIBUTING.md](./CONTRIBUTING.md)\n\n## 👏 Contributors\n\nShout out to these awesome contributors:\n\n\u003ca href=\"https://github.com/awslabs/agent-evaluation/graphs/contributors\"\u003e\n  \u003cimg src=\"https://contrib.rocks/image?repo=awslabs/agent-evaluation\" /\u003e\n\u003c/a\u003e","project_url":"https://awesome.ecosyste.ms/api/v1/projects/awslabs.github.io%2Fagent-evaluation%2F","html_url":"https://awesome.ecosyste.ms/projects/awslabs.github.io%2Fagent-evaluation%2F","lists_url":"https://awesome.ecosyste.ms/api/v1/projects/awslabs.github.io%2Fagent-evaluation%2F/lists"}