{"id":22677199,"url":"https://github.com/daemon/parrot","last_synced_at":"2025-03-29T12:44:40.154Z","repository":{"id":251288759,"uuid":"836964909","full_name":"daemon/parrot","owner":"daemon","description":"Fluent LLM evaluation framework.","archived":false,"fork":false,"pushed_at":"2024-08-02T19:03:24.000Z","size":43,"stargazers_count":0,"open_issues_count":0,"forks_count":0,"subscribers_count":1,"default_branch":"master","last_synced_at":"2025-02-04T13:43:43.298Z","etag":null,"topics":[],"latest_commit_sha":null,"homepage":null,"language":"Python","has_issues":true,"has_wiki":null,"has_pages":null,"mirror_url":null,"source_name":null,"license":"mit","status":null,"scm":"git","pull_requests_enabled":true,"icon_url":"https://github.com/daemon.png","metadata":{"files":{"readme":"README.md","changelog":null,"contributing":null,"funding":null,"license":"LICENSE","code_of_conduct":null,"threat_model":null,"audit":null,"citation":null,"codeowners":null,"security":null,"support":null,"governance":null,"roadmap":null,"authors":null,"dei":null,"publiccode":null,"codemeta":null}},"created_at":"2024-08-01T23:44:53.000Z","updated_at":"2024-09-18T14:14:48.000Z","dependencies_parsed_at":"2024-08-02T01:14:09.645Z","dependency_job_id":"dee09372-bd04-4712-a160-97a4a3c3d8d7","html_url":"https://github.com/daemon/parrot","commit_stats":null,"previous_names":["daemon/parrot"],"tags_count":0,"template":false,"template_full_name":null,"repository_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/daemon%2Fparrot","tags_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/daemon%2Fparrot/tags","releases_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/daemon%2Fparrot/releases","manifests_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/daemon%2Fparrot/manifests","owner_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners/daemon","download_url":"https://codeload.github.com/daemon/parrot/tar.gz/refs/heads/master","host":{"name":"GitHub","url":"https://github.com","kind":"github","repositories_count":246187218,"owners_count":20737459,"icon_url":"https://github.com/github.png","version":null,"created_at":"2022-05-30T11:31:42.601Z","updated_at":"2022-07-04T15:15:14.044Z","host_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub","repositories_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories","repository_names_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repository_names","owners_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners"}},"keywords":[],"created_at":"2024-12-09T17:59:15.602Z","updated_at":"2025-03-29T12:44:40.131Z","avatar_url":"https://github.com/daemon.png","language":"Python","funding_links":[],"categories":[],"sub_categories":[],"readme":"# Parrot: A Fluent LLM Evaluation Framework\r\n\r\n## Getting Started\r\n\r\n### Quick Start\r\n\r\n1. For a non-editable version: `pip install git+https://github.com/daemon/parrot`\r\n2. For an editable version, clone the repository and install it with `pip install -e .`\r\n3.\r\n\r\n```python\r\nimport argparse\r\nimport asyncio\r\n\r\nimport parrot\r\nimport parrot.tasks\r\n\r\n\r\nasync def amain():\r\n    parser = argparse.ArgumentParser()\r\n    parser.add_argument('--output-folder', '-o', type=str, default='tech-results')\r\n    parser.add_argument('--num-rows-per-task', '-n', type=int, default=20)\r\n    args = parser.parse_args()\r\n\r\n    gpt_4o_llm = parrot.AzureOpenAILLM(parrot.AzureOpenAICredentials.parse_file('gpt4o.json'))\r\n    gpt_35_llm = parrot.AzureOpenAILLM(parrot.AzureOpenAICredentials.parse_file('gpt35.json'))\r\n\r\n    with gpt_4o_llm.as_evaluator('profiles/tech.json') as evaluator:\r\n        evaluator.add_task(parrot.tasks.QuestionAnswering(\r\n            subcategory='retrocomputing',\r\n            name='hard software q\u0026a',\r\n            num_rows=args.num_rows_per_task,\r\n            difficulty='nearly impossible',\r\n        ))\r\n\r\n        evaluator.add_task(parrot.tasks.QuestionAnswering(\r\n            subcategory='retrocomputing',\r\n            name='easy software q\u0026a',\r\n            num_rows=args.num_rows_per_task,\r\n            difficulty='very easy',\r\n        ))\r\n\r\n        # Add any kind of kwargs to build the task\r\n        evaluator.add_task(parrot.tasks.SentimentClassification(\r\n            subcategory='web services',\r\n            creativity='very high',\r\n            num_rows=args.num_rows_per_task,\r\n            name='creative web services sentiment classification',\r\n            difficulty='very challenging'\r\n        ))\r\n\r\n        evaluator.add_task(parrot.tasks.DocumentClassification(\r\n            subcategory='web services',\r\n            creativity='very high',\r\n            literary_difficulty='very challenging',\r\n            long_prose=True,\r\n            num_rows=args.num_rows_per_task,\r\n            name='creative web services document classification',\r\n            difficulty='very challenging'\r\n        ))\r\n\r\n        evaluator.add_task(parrot.tasks.StoryGeneration(\r\n            name='creative hardware text generation',\r\n            subcategory='hardware',\r\n            num_rows=args.num_rows_per_task,\r\n            difficulty='extremely difficult and collegial'\r\n        ))\r\n\r\n        # Run the evaluation\r\n        results = await evaluator.evaluate(gpt_35_llm)\r\n\r\n    # evaluator.add_task(parrot.tasks.QuestionAnswering())  # Error! Immutable evaluator for safety reasons\r\n    print(results)\r\n\r\n    results.save(args.output_folder)\r\n\r\n\r\ndef main():\r\n    asyncio.run(amain())\r\n\r\n\r\nif __name__ == '__main__':\r\n    main()\r\n```\r\n","project_url":"https://awesome.ecosyste.ms/api/v1/projects/github.com%2Fdaemon%2Fparrot","html_url":"https://awesome.ecosyste.ms/projects/github.com%2Fdaemon%2Fparrot","lists_url":"https://awesome.ecosyste.ms/api/v1/projects/github.com%2Fdaemon%2Fparrot/lists"}