{"id":26638045,"url":"https://github.com/spellcraftai/oaib","last_synced_at":"2025-07-06T17:03:42.114Z","repository":{"id":221856781,"uuid":"755561103","full_name":"SpellcraftAI/oaib","owner":"SpellcraftAI","description":"Use the OpenAI Batch tool to make async batch requests to the OpenAI API.","archived":false,"fork":false,"pushed_at":"2024-03-02T22:26:29.000Z","size":88,"stargazers_count":96,"open_issues_count":6,"forks_count":3,"subscribers_count":3,"default_branch":"dev","last_synced_at":"2025-03-24T17:28:22.519Z","etag":null,"topics":[],"latest_commit_sha":null,"homepage":"","language":"Python","has_issues":true,"has_wiki":null,"has_pages":null,"mirror_url":null,"source_name":null,"license":"mit","status":null,"scm":"git","pull_requests_enabled":true,"icon_url":"https://github.com/SpellcraftAI.png","metadata":{"files":{"readme":"README.ipynb","changelog":null,"contributing":null,"funding":null,"license":"LICENSE","code_of_conduct":null,"threat_model":null,"audit":null,"citation":null,"codeowners":null,"security":null,"support":null,"governance":null,"roadmap":null,"authors":null,"dei":null}},"created_at":"2024-02-10T15:35:29.000Z","updated_at":"2025-03-19T04:43:36.000Z","dependencies_parsed_at":"2024-03-02T23:20:57.610Z","dependency_job_id":"58e433d2-f9b2-48b3-8b72-f3ec9e7bff19","html_url":"https://github.com/SpellcraftAI/oaib","commit_stats":null,"previous_names":["spellcraftai/oaib"],"tags_count":10,"template":false,"template_full_name":null,"repository_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/SpellcraftAI%2Foaib","tags_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/SpellcraftAI%2Foaib/tags","releases_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/SpellcraftAI%2Foaib/releases","manifests_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/SpellcraftAI%2Foaib/manifests","owner_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners/SpellcraftAI","download_url":"https://codeload.github.com/SpellcraftAI/oaib/tar.gz/refs/heads/dev","host":{"name":"GitHub","url":"https://github.com","kind":"github","repositories_count":248283170,"owners_count":21077788,"icon_url":"https://github.com/github.png","version":null,"created_at":"2022-05-30T11:31:42.601Z","updated_at":"2022-07-04T15:15:14.044Z","host_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub","repositories_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories","repository_names_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repository_names","owners_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners"}},"keywords":[],"created_at":"2025-03-24T17:24:58.430Z","updated_at":"2025-04-10T19:50:51.700Z","avatar_url":"https://github.com/SpellcraftAI.png","language":"Python","funding_links":[],"categories":[],"sub_categories":[],"readme":"{\n \"cells\": [\n  {\n   \"cell_type\": \"markdown\",\n   \"metadata\": {},\n   \"source\": [\n    \"# OpenAI Batcher (`oaib`)\\n\",\n    \"\\n\",\n    \"A Python library for making rate-limited, async batch requests to the OpenAI\\n\",\n    \"API.\\n\",\n    \"\\n\",\n    \"- Use `Batch` to make batch requests as quickly as possible given TPM/RPM\\n\",\n    \"  limits.\\n\",\n    \"- Use `Auto` to automatically read your rate limits from OpenAI's response\\n\",\n    \"  headers, and run the job as fast as possible.\"\n   ]\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"metadata\": {},\n   \"source\": [\n    \"### Set OpenAI API Key\"\n   ]\n  },\n  {\n   \"cell_type\": \"code\",\n   \"execution_count\": 2,\n   \"metadata\": {},\n   \"outputs\": [],\n   \"source\": [\n    \"import os\\n\",\n    \"os.environ['OPENAI_API_KEY'] = input()\"\n   ]\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"metadata\": {},\n   \"source\": [\n    \"### Using the `Batch` class\\n\",\n    \"\\n\",\n    \"You can mix and match endpoints as needed for regular `Batch`, but it's not\\n\",\n    \"recommended as the rate limits for different endpoints/models will differ. For\\n\",\n    \"maximum efficiency, use `Auto`.\"\n   ]\n  },\n  {\n   \"cell_type\": \"code\",\n   \"execution_count\": 5,\n   \"metadata\": {},\n   \"outputs\": [\n    {\n     \"data\": {\n      \"application/vnd.jupyter.widget-view+json\": {\n       \"model_id\": \"1b639118a1114517bb39abdae101cfcf\",\n       \"version_major\": 2,\n       \"version_minor\": 0\n      },\n      \"text/plain\": [\n       \"  0%|          | 0/20 [00:00\u003c?, ?req/s]\"\n      ]\n     },\n     \"metadata\": {},\n     \"output_type\": \"display_data\"\n    },\n    {\n     \"data\": {\n      \"application/vnd.jupyter.widget-view+json\": {\n       \"model_id\": \"691da1fccbf1427ca8be81548594d5d6\",\n       \"version_major\": 2,\n       \"version_minor\": 0\n      },\n      \"text/plain\": [\n       \"RPM:   0%|          | 0/100\"\n      ]\n     },\n     \"metadata\": {},\n     \"output_type\": \"display_data\"\n    },\n    {\n     \"data\": {\n      \"application/vnd.jupyter.widget-view+json\": {\n       \"model_id\": \"74c6cefa195a4bf89a743ff6891e207b\",\n       \"version_major\": 2,\n       \"version_minor\": 0\n      },\n      \"text/plain\": [\n       \"TPM:   0%|          | 0/1000\"\n      ]\n     },\n     \"metadata\": {},\n     \"output_type\": \"display_data\"\n    },\n    {\n     \"name\": \"stdout\",\n     \"output_type\": \"stream\",\n     \"text\": [\n      \"\\n\",\n      \"Run took 20.02s.\\n\",\n      \"\\n\"\n     ]\n    },\n    {\n     \"data\": {\n      \"text/html\": [\n       \"\u003cdiv\u003e\\n\",\n       \"\u003cstyle scoped\u003e\\n\",\n       \"    .dataframe tbody tr th:only-of-type {\\n\",\n       \"        vertical-align: middle;\\n\",\n       \"    }\\n\",\n       \"\\n\",\n       \"    .dataframe tbody tr th {\\n\",\n       \"        vertical-align: top;\\n\",\n       \"    }\\n\",\n       \"\\n\",\n       \"    .dataframe thead th {\\n\",\n       \"        text-align: right;\\n\",\n       \"    }\\n\",\n       \"\u003c/style\u003e\\n\",\n       \"\u003ctable border=\\\"1\\\" class=\\\"dataframe\\\"\u003e\\n\",\n       \"  \u003cthead\u003e\\n\",\n       \"    \u003ctr style=\\\"text-align: right;\\\"\u003e\\n\",\n       \"      \u003cth\u003e\u003c/th\u003e\\n\",\n       \"      \u003cth\u003eendpoint\u003c/th\u003e\\n\",\n       \"      \u003cth\u003emodel\u003c/th\u003e\\n\",\n       \"      \u003cth\u003emessages\u003c/th\u003e\\n\",\n       \"      \u003cth\u003eresult\u003c/th\u003e\\n\",\n       \"    \u003c/tr\u003e\\n\",\n       \"  \u003c/thead\u003e\\n\",\n       \"  \u003ctbody\u003e\\n\",\n       \"    \u003ctr\u003e\\n\",\n       \"      \u003cth\u003e0\u003c/th\u003e\\n\",\n       \"      \u003ctd\u003echat.completions.create\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003egpt-3.5-turbo\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003e[{'role': 'user', 'content': 'say hello'}]\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003e{'id': 'chatcmpl-8qVzMf7oXHlLWpsISVpAaUPxps0g5...\u003c/td\u003e\\n\",\n       \"    \u003c/tr\u003e\\n\",\n       \"    \u003ctr\u003e\\n\",\n       \"      \u003cth\u003e1\u003c/th\u003e\\n\",\n       \"      \u003ctd\u003echat.completions.create\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003egpt-3.5-turbo\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003e[{'role': 'user', 'content': 'say hello'}]\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003e{'id': 'chatcmpl-8qVzNeFY9TLN71o2FMEqGssHIrOQq...\u003c/td\u003e\\n\",\n       \"    \u003c/tr\u003e\\n\",\n       \"    \u003ctr\u003e\\n\",\n       \"      \u003cth\u003e2\u003c/th\u003e\\n\",\n       \"      \u003ctd\u003echat.completions.create\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003egpt-3.5-turbo\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003e[{'role': 'user', 'content': 'say hello'}]\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003e{'id': 'chatcmpl-8qVzNDiD3ikFtBZ4hHXWEsLONHUeS...\u003c/td\u003e\\n\",\n       \"    \u003c/tr\u003e\\n\",\n       \"    \u003ctr\u003e\\n\",\n       \"      \u003cth\u003e3\u003c/th\u003e\\n\",\n       \"      \u003ctd\u003echat.completions.create\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003egpt-3.5-turbo\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003e[{'role': 'user', 'content': 'say hello'}]\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003e{'id': 'chatcmpl-8qVzP9dccsvrGsOR3X5HgmHqsR2fm...\u003c/td\u003e\\n\",\n       \"    \u003c/tr\u003e\\n\",\n       \"    \u003ctr\u003e\\n\",\n       \"      \u003cth\u003e4\u003c/th\u003e\\n\",\n       \"      \u003ctd\u003echat.completions.create\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003egpt-3.5-turbo\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003e[{'role': 'user', 'content': 'say hello'}]\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003e{'id': 'chatcmpl-8qVzQV7ZnIoXccx9R8dIfS4rdPd0U...\u003c/td\u003e\\n\",\n       \"    \u003c/tr\u003e\\n\",\n       \"    \u003ctr\u003e\\n\",\n       \"      \u003cth\u003e5\u003c/th\u003e\\n\",\n       \"      \u003ctd\u003echat.completions.create\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003egpt-3.5-turbo\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003e[{'role': 'user', 'content': 'say hello'}]\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003e{'id': 'chatcmpl-8qVzRVQvp3wwmEvbFzNPtrXBmcOhR...\u003c/td\u003e\\n\",\n       \"    \u003c/tr\u003e\\n\",\n       \"    \u003ctr\u003e\\n\",\n       \"      \u003cth\u003e6\u003c/th\u003e\\n\",\n       \"      \u003ctd\u003echat.completions.create\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003egpt-3.5-turbo\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003e[{'role': 'user', 'content': 'say hello'}]\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003e{'id': 'chatcmpl-8qVzRSw7iTCLs0uu8fWZwDcaPGB0s...\u003c/td\u003e\\n\",\n       \"    \u003c/tr\u003e\\n\",\n       \"    \u003ctr\u003e\\n\",\n       \"      \u003cth\u003e7\u003c/th\u003e\\n\",\n       \"      \u003ctd\u003echat.completions.create\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003egpt-3.5-turbo\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003e[{'role': 'user', 'content': 'say hello'}]\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003e{'id': 'chatcmpl-8qVzS6D1gACsJW6JXvuS42N4lQLh7...\u003c/td\u003e\\n\",\n       \"    \u003c/tr\u003e\\n\",\n       \"    \u003ctr\u003e\\n\",\n       \"      \u003cth\u003e8\u003c/th\u003e\\n\",\n       \"      \u003ctd\u003echat.completions.create\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003egpt-3.5-turbo\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003e[{'role': 'user', 'content': 'say hello'}]\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003e{'id': 'chatcmpl-8qVzUFmpIzWjKsNGnlLvZW3DhF752...\u003c/td\u003e\\n\",\n       \"    \u003c/tr\u003e\\n\",\n       \"    \u003ctr\u003e\\n\",\n       \"      \u003cth\u003e9\u003c/th\u003e\\n\",\n       \"      \u003ctd\u003echat.completions.create\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003egpt-3.5-turbo\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003e[{'role': 'user', 'content': 'say hello'}]\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003e{'id': 'chatcmpl-8qVzU6Mg6Zk4BC5uelndmHjmGAQ0I...\u003c/td\u003e\\n\",\n       \"    \u003c/tr\u003e\\n\",\n       \"    \u003ctr\u003e\\n\",\n       \"      \u003cth\u003e10\u003c/th\u003e\\n\",\n       \"      \u003ctd\u003echat.completions.create\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003egpt-3.5-turbo\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003e[{'role': 'user', 'content': 'say hello'}]\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003e{'id': 'chatcmpl-8qVzV5K4OEk80dDuSwohiTualLOoO...\u003c/td\u003e\\n\",\n       \"    \u003c/tr\u003e\\n\",\n       \"    \u003ctr\u003e\\n\",\n       \"      \u003cth\u003e11\u003c/th\u003e\\n\",\n       \"      \u003ctd\u003echat.completions.create\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003egpt-3.5-turbo\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003e[{'role': 'user', 'content': 'say hello'}]\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003e{'id': 'chatcmpl-8qVzXCcmbcuy1EQPskJrfN5po1Ix9...\u003c/td\u003e\\n\",\n       \"    \u003c/tr\u003e\\n\",\n       \"    \u003ctr\u003e\\n\",\n       \"      \u003cth\u003e12\u003c/th\u003e\\n\",\n       \"      \u003ctd\u003echat.completions.create\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003egpt-3.5-turbo\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003e[{'role': 'user', 'content': 'say hello'}]\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003e{'id': 'chatcmpl-8qVzX4UfaBPDO3fF8vMO1dsQ6tfiT...\u003c/td\u003e\\n\",\n       \"    \u003c/tr\u003e\\n\",\n       \"    \u003ctr\u003e\\n\",\n       \"      \u003cth\u003e13\u003c/th\u003e\\n\",\n       \"      \u003ctd\u003echat.completions.create\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003egpt-3.5-turbo\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003e[{'role': 'user', 'content': 'say hello'}]\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003e{'id': 'chatcmpl-8qVzZme2VBNhckfItEZRJqmpC3E53...\u003c/td\u003e\\n\",\n       \"    \u003c/tr\u003e\\n\",\n       \"    \u003ctr\u003e\\n\",\n       \"      \u003cth\u003e14\u003c/th\u003e\\n\",\n       \"      \u003ctd\u003echat.completions.create\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003egpt-3.5-turbo\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003e[{'role': 'user', 'content': 'say hello'}]\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003e{'id': 'chatcmpl-8qVzaYddjxrFpmYUDjMHXlDPgS7G4...\u003c/td\u003e\\n\",\n       \"    \u003c/tr\u003e\\n\",\n       \"    \u003ctr\u003e\\n\",\n       \"      \u003cth\u003e15\u003c/th\u003e\\n\",\n       \"      \u003ctd\u003echat.completions.create\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003egpt-3.5-turbo\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003e[{'role': 'user', 'content': 'say hello'}]\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003e{'id': 'chatcmpl-8qVzcTLFClvwjtPNNyv3KJ2xvZ8dQ...\u003c/td\u003e\\n\",\n       \"    \u003c/tr\u003e\\n\",\n       \"    \u003ctr\u003e\\n\",\n       \"      \u003cth\u003e16\u003c/th\u003e\\n\",\n       \"      \u003ctd\u003echat.completions.create\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003egpt-3.5-turbo\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003e[{'role': 'user', 'content': 'say hello'}]\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003e{'id': 'chatcmpl-8qVzcuWlMUwqXj0AeMQkZqzNIQvJo...\u003c/td\u003e\\n\",\n       \"    \u003c/tr\u003e\\n\",\n       \"    \u003ctr\u003e\\n\",\n       \"      \u003cth\u003e17\u003c/th\u003e\\n\",\n       \"      \u003ctd\u003echat.completions.create\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003egpt-3.5-turbo\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003e[{'role': 'user', 'content': 'say hello'}]\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003e{'id': 'chatcmpl-8qVzepfAhFnZj1AlnVSyNHHMyZ0mK...\u003c/td\u003e\\n\",\n       \"    \u003c/tr\u003e\\n\",\n       \"    \u003ctr\u003e\\n\",\n       \"      \u003cth\u003e18\u003c/th\u003e\\n\",\n       \"      \u003ctd\u003echat.completions.create\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003egpt-3.5-turbo\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003e[{'role': 'user', 'content': 'say hello'}]\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003e{'id': 'chatcmpl-8qVzfdULvZx6OyUZpARpTMNKYzfKx...\u003c/td\u003e\\n\",\n       \"    \u003c/tr\u003e\\n\",\n       \"    \u003ctr\u003e\\n\",\n       \"      \u003cth\u003e19\u003c/th\u003e\\n\",\n       \"      \u003ctd\u003echat.completions.create\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003egpt-3.5-turbo\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003e[{'role': 'user', 'content': 'say hello'}]\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003e{'id': 'chatcmpl-8qVzfyEru2f67JyunYCdPMaC0fD2H...\u003c/td\u003e\\n\",\n       \"    \u003c/tr\u003e\\n\",\n       \"  \u003c/tbody\u003e\\n\",\n       \"\u003c/table\u003e\\n\",\n       \"\u003c/div\u003e\"\n      ],\n      \"text/plain\": [\n       \"                   endpoint          model  \\\\\\n\",\n       \"0   chat.completions.create  gpt-3.5-turbo   \\n\",\n       \"1   chat.completions.create  gpt-3.5-turbo   \\n\",\n       \"2   chat.completions.create  gpt-3.5-turbo   \\n\",\n       \"3   chat.completions.create  gpt-3.5-turbo   \\n\",\n       \"4   chat.completions.create  gpt-3.5-turbo   \\n\",\n       \"5   chat.completions.create  gpt-3.5-turbo   \\n\",\n       \"6   chat.completions.create  gpt-3.5-turbo   \\n\",\n       \"7   chat.completions.create  gpt-3.5-turbo   \\n\",\n       \"8   chat.completions.create  gpt-3.5-turbo   \\n\",\n       \"9   chat.completions.create  gpt-3.5-turbo   \\n\",\n       \"10  chat.completions.create  gpt-3.5-turbo   \\n\",\n       \"11  chat.completions.create  gpt-3.5-turbo   \\n\",\n       \"12  chat.completions.create  gpt-3.5-turbo   \\n\",\n       \"13  chat.completions.create  gpt-3.5-turbo   \\n\",\n       \"14  chat.completions.create  gpt-3.5-turbo   \\n\",\n       \"15  chat.completions.create  gpt-3.5-turbo   \\n\",\n       \"16  chat.completions.create  gpt-3.5-turbo   \\n\",\n       \"17  chat.completions.create  gpt-3.5-turbo   \\n\",\n       \"18  chat.completions.create  gpt-3.5-turbo   \\n\",\n       \"19  chat.completions.create  gpt-3.5-turbo   \\n\",\n       \"\\n\",\n       \"                                      messages  \\\\\\n\",\n       \"0   [{'role': 'user', 'content': 'say hello'}]   \\n\",\n       \"1   [{'role': 'user', 'content': 'say hello'}]   \\n\",\n       \"2   [{'role': 'user', 'content': 'say hello'}]   \\n\",\n       \"3   [{'role': 'user', 'content': 'say hello'}]   \\n\",\n       \"4   [{'role': 'user', 'content': 'say hello'}]   \\n\",\n       \"5   [{'role': 'user', 'content': 'say hello'}]   \\n\",\n       \"6   [{'role': 'user', 'content': 'say hello'}]   \\n\",\n       \"7   [{'role': 'user', 'content': 'say hello'}]   \\n\",\n       \"8   [{'role': 'user', 'content': 'say hello'}]   \\n\",\n       \"9   [{'role': 'user', 'content': 'say hello'}]   \\n\",\n       \"10  [{'role': 'user', 'content': 'say hello'}]   \\n\",\n       \"11  [{'role': 'user', 'content': 'say hello'}]   \\n\",\n       \"12  [{'role': 'user', 'content': 'say hello'}]   \\n\",\n       \"13  [{'role': 'user', 'content': 'say hello'}]   \\n\",\n       \"14  [{'role': 'user', 'content': 'say hello'}]   \\n\",\n       \"15  [{'role': 'user', 'content': 'say hello'}]   \\n\",\n       \"16  [{'role': 'user', 'content': 'say hello'}]   \\n\",\n       \"17  [{'role': 'user', 'content': 'say hello'}]   \\n\",\n       \"18  [{'role': 'user', 'content': 'say hello'}]   \\n\",\n       \"19  [{'role': 'user', 'content': 'say hello'}]   \\n\",\n       \"\\n\",\n       \"                                               result  \\n\",\n       \"0   {'id': 'chatcmpl-8qVzMf7oXHlLWpsISVpAaUPxps0g5...  \\n\",\n       \"1   {'id': 'chatcmpl-8qVzNeFY9TLN71o2FMEqGssHIrOQq...  \\n\",\n       \"2   {'id': 'chatcmpl-8qVzNDiD3ikFtBZ4hHXWEsLONHUeS...  \\n\",\n       \"3   {'id': 'chatcmpl-8qVzP9dccsvrGsOR3X5HgmHqsR2fm...  \\n\",\n       \"4   {'id': 'chatcmpl-8qVzQV7ZnIoXccx9R8dIfS4rdPd0U...  \\n\",\n       \"5   {'id': 'chatcmpl-8qVzRVQvp3wwmEvbFzNPtrXBmcOhR...  \\n\",\n       \"6   {'id': 'chatcmpl-8qVzRSw7iTCLs0uu8fWZwDcaPGB0s...  \\n\",\n       \"7   {'id': 'chatcmpl-8qVzS6D1gACsJW6JXvuS42N4lQLh7...  \\n\",\n       \"8   {'id': 'chatcmpl-8qVzUFmpIzWjKsNGnlLvZW3DhF752...  \\n\",\n       \"9   {'id': 'chatcmpl-8qVzU6Mg6Zk4BC5uelndmHjmGAQ0I...  \\n\",\n       \"10  {'id': 'chatcmpl-8qVzV5K4OEk80dDuSwohiTualLOoO...  \\n\",\n       \"11  {'id': 'chatcmpl-8qVzXCcmbcuy1EQPskJrfN5po1Ix9...  \\n\",\n       \"12  {'id': 'chatcmpl-8qVzX4UfaBPDO3fF8vMO1dsQ6tfiT...  \\n\",\n       \"13  {'id': 'chatcmpl-8qVzZme2VBNhckfItEZRJqmpC3E53...  \\n\",\n       \"14  {'id': 'chatcmpl-8qVzaYddjxrFpmYUDjMHXlDPgS7G4...  \\n\",\n       \"15  {'id': 'chatcmpl-8qVzcTLFClvwjtPNNyv3KJ2xvZ8dQ...  \\n\",\n       \"16  {'id': 'chatcmpl-8qVzcuWlMUwqXj0AeMQkZqzNIQvJo...  \\n\",\n       \"17  {'id': 'chatcmpl-8qVzepfAhFnZj1AlnVSyNHHMyZ0mK...  \\n\",\n       \"18  {'id': 'chatcmpl-8qVzfdULvZx6OyUZpARpTMNKYzfKx...  \\n\",\n       \"19  {'id': 'chatcmpl-8qVzfyEru2f67JyunYCdPMaC0fD2H...  \"\n      ]\n     },\n     \"execution_count\": 5,\n     \"metadata\": {},\n     \"output_type\": \"execute_result\"\n    }\n   ],\n   \"source\": [\n    \"from oaib import Batch\\n\",\n    \"\\n\",\n    \"# Use low rate limits for this example.\\n\",\n    \"batch = Batch(rpm=100, tpm=1_000, workers=5)\\n\",\n    \"\\n\",\n    \"# Creating a batch with 20 chat completions.\\n\",\n    \"for i in range(20):\\n\",\n    \"    await batch.add(\\n\",\n    \"        \\\"chat.completions.create\\\", \\n\",\n    \"        model=\\\"gpt-3.5-turbo\\\", \\n\",\n    \"        messages=[{\\\"role\\\": \\\"user\\\", \\\"content\\\": \\\"say hello\\\"}]\\n\",\n    \"    )\\n\",\n    \"\\n\",\n    \"await batch.run()\"\n   ]\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"metadata\": {},\n   \"source\": [\n    \"### Using the `Auto` class\\n\",\n    \"\\n\",\n    \"Automatically use the given TPM/RPM rate limits provided by OpenAI API\\n\",\n    \"responses.\"\n   ]\n  },\n  {\n   \"cell_type\": \"code\",\n   \"execution_count\": 4,\n   \"metadata\": {},\n   \"outputs\": [\n    {\n     \"data\": {\n      \"application/vnd.jupyter.widget-view+json\": {\n       \"model_id\": \"67d8275d5a2f4f9cbd2ab2eef1fd1c0d\",\n       \"version_major\": 2,\n       \"version_minor\": 0\n      },\n      \"text/plain\": [\n       \"  0%|          | 0/1000 [00:00\u003c?, ?req/s]\"\n      ]\n     },\n     \"metadata\": {},\n     \"output_type\": \"display_data\"\n    },\n    {\n     \"data\": {\n      \"application/vnd.jupyter.widget-view+json\": {\n       \"model_id\": \"034506ef02c841f59533536f425e5eb6\",\n       \"version_major\": 2,\n       \"version_minor\": 0\n      },\n      \"text/plain\": [\n       \"RPM:   0%|          | 0/500\"\n      ]\n     },\n     \"metadata\": {},\n     \"output_type\": \"display_data\"\n    },\n    {\n     \"data\": {\n      \"application/vnd.jupyter.widget-view+json\": {\n       \"model_id\": \"64443f4ef537420389abd7ceb2485cef\",\n       \"version_major\": 2,\n       \"version_minor\": 0\n      },\n      \"text/plain\": [\n       \"TPM:   0%|          | 0/10000\"\n      ]\n     },\n     \"metadata\": {},\n     \"output_type\": \"display_data\"\n    },\n    {\n     \"name\": \"stdout\",\n     \"output_type\": \"stream\",\n     \"text\": [\n      \"\\n\",\n      \"Run took 12.58s.\\n\",\n      \"\\n\"\n     ]\n    },\n    {\n     \"data\": {\n      \"text/html\": [\n       \"\u003cdiv\u003e\\n\",\n       \"\u003cstyle scoped\u003e\\n\",\n       \"    .dataframe tbody tr th:only-of-type {\\n\",\n       \"        vertical-align: middle;\\n\",\n       \"    }\\n\",\n       \"\\n\",\n       \"    .dataframe tbody tr th {\\n\",\n       \"        vertical-align: top;\\n\",\n       \"    }\\n\",\n       \"\\n\",\n       \"    .dataframe thead th {\\n\",\n       \"        text-align: right;\\n\",\n       \"    }\\n\",\n       \"\u003c/style\u003e\\n\",\n       \"\u003ctable border=\\\"1\\\" class=\\\"dataframe\\\"\u003e\\n\",\n       \"  \u003cthead\u003e\\n\",\n       \"    \u003ctr style=\\\"text-align: right;\\\"\u003e\\n\",\n       \"      \u003cth\u003e\u003c/th\u003e\\n\",\n       \"      \u003cth\u003eendpoint\u003c/th\u003e\\n\",\n       \"      \u003cth\u003emodel\u003c/th\u003e\\n\",\n       \"      \u003cth\u003emessages\u003c/th\u003e\\n\",\n       \"      \u003cth\u003eresult\u003c/th\u003e\\n\",\n       \"    \u003c/tr\u003e\\n\",\n       \"  \u003c/thead\u003e\\n\",\n       \"  \u003ctbody\u003e\\n\",\n       \"    \u003ctr\u003e\\n\",\n       \"      \u003cth\u003e0\u003c/th\u003e\\n\",\n       \"      \u003ctd\u003echat.completions.create\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003egpt-3.5-turbo\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003e[{'role': 'user', 'content': 'say hello'}]\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003e{'id': 'chatcmpl-8qVxz211nsiQQSY2k54r4r141UX83...\u003c/td\u003e\\n\",\n       \"    \u003c/tr\u003e\\n\",\n       \"    \u003ctr\u003e\\n\",\n       \"      \u003cth\u003e1\u003c/th\u003e\\n\",\n       \"      \u003ctd\u003echat.completions.create\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003egpt-3.5-turbo\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003e[{'role': 'user', 'content': 'say hello'}]\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003e{'id': 'chatcmpl-8qVy0TrnGax3XqkDOlEiImosPnvIL...\u003c/td\u003e\\n\",\n       \"    \u003c/tr\u003e\\n\",\n       \"    \u003ctr\u003e\\n\",\n       \"      \u003cth\u003e2\u003c/th\u003e\\n\",\n       \"      \u003ctd\u003echat.completions.create\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003egpt-3.5-turbo\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003e[{'role': 'user', 'content': 'say hello'}]\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003e{'id': 'chatcmpl-8qVxzMdsF5v1je6iAmvjK7BCZKQna...\u003c/td\u003e\\n\",\n       \"    \u003c/tr\u003e\\n\",\n       \"    \u003ctr\u003e\\n\",\n       \"      \u003cth\u003e3\u003c/th\u003e\\n\",\n       \"      \u003ctd\u003echat.completions.create\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003egpt-3.5-turbo\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003e[{'role': 'user', 'content': 'say hello'}]\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003e{'id': 'chatcmpl-8qVy0KiRWYPtQ099p3b1k1HfYBYwT...\u003c/td\u003e\\n\",\n       \"    \u003c/tr\u003e\\n\",\n       \"    \u003ctr\u003e\\n\",\n       \"      \u003cth\u003e4\u003c/th\u003e\\n\",\n       \"      \u003ctd\u003echat.completions.create\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003egpt-3.5-turbo\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003e[{'role': 'user', 'content': 'say hello'}]\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003e{'id': 'chatcmpl-8qVy0M7l6Fg0mvWCpOXomVSqV6Pow...\u003c/td\u003e\\n\",\n       \"    \u003c/tr\u003e\\n\",\n       \"    \u003ctr\u003e\\n\",\n       \"      \u003cth\u003e...\u003c/th\u003e\\n\",\n       \"      \u003ctd\u003e...\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003e...\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003e...\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003e...\u003c/td\u003e\\n\",\n       \"    \u003c/tr\u003e\\n\",\n       \"    \u003ctr\u003e\\n\",\n       \"      \u003cth\u003e995\u003c/th\u003e\\n\",\n       \"      \u003ctd\u003echat.completions.create\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003egpt-3.5-turbo\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003e[{'role': 'user', 'content': 'say hello'}]\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003e{'id': 'chatcmpl-8qVyBemEPgb6lV5Opnu8X9UQ7T9iZ...\u003c/td\u003e\\n\",\n       \"    \u003c/tr\u003e\\n\",\n       \"    \u003ctr\u003e\\n\",\n       \"      \u003cth\u003e996\u003c/th\u003e\\n\",\n       \"      \u003ctd\u003echat.completions.create\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003egpt-3.5-turbo\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003e[{'role': 'user', 'content': 'say hello'}]\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003e{'id': 'chatcmpl-8qVyBBlh4PMbI9qtca80UyMbrOGAF...\u003c/td\u003e\\n\",\n       \"    \u003c/tr\u003e\\n\",\n       \"    \u003ctr\u003e\\n\",\n       \"      \u003cth\u003e997\u003c/th\u003e\\n\",\n       \"      \u003ctd\u003echat.completions.create\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003egpt-3.5-turbo\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003e[{'role': 'user', 'content': 'say hello'}]\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003e{'id': 'chatcmpl-8qVyBWveGsMeSLS5SzUbXIiMLvGaS...\u003c/td\u003e\\n\",\n       \"    \u003c/tr\u003e\\n\",\n       \"    \u003ctr\u003e\\n\",\n       \"      \u003cth\u003e998\u003c/th\u003e\\n\",\n       \"      \u003ctd\u003echat.completions.create\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003egpt-3.5-turbo\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003e[{'role': 'user', 'content': 'say hello'}]\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003e{'id': 'chatcmpl-8qVyBbBnX5SleJWSDIHqZ8lS0y15V...\u003c/td\u003e\\n\",\n       \"    \u003c/tr\u003e\\n\",\n       \"    \u003ctr\u003e\\n\",\n       \"      \u003cth\u003e999\u003c/th\u003e\\n\",\n       \"      \u003ctd\u003echat.completions.create\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003egpt-3.5-turbo\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003e[{'role': 'user', 'content': 'say hello'}]\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003e{'id': 'chatcmpl-8qVyBYTizFbBIu3RQ9XlvKDlnSuEG...\u003c/td\u003e\\n\",\n       \"    \u003c/tr\u003e\\n\",\n       \"  \u003c/tbody\u003e\\n\",\n       \"\u003c/table\u003e\\n\",\n       \"\u003cp\u003e1000 rows × 4 columns\u003c/p\u003e\\n\",\n       \"\u003c/div\u003e\"\n      ],\n      \"text/plain\": [\n       \"                    endpoint          model  \\\\\\n\",\n       \"0    chat.completions.create  gpt-3.5-turbo   \\n\",\n       \"1    chat.completions.create  gpt-3.5-turbo   \\n\",\n       \"2    chat.completions.create  gpt-3.5-turbo   \\n\",\n       \"3    chat.completions.create  gpt-3.5-turbo   \\n\",\n       \"4    chat.completions.create  gpt-3.5-turbo   \\n\",\n       \"..                       ...            ...   \\n\",\n       \"995  chat.completions.create  gpt-3.5-turbo   \\n\",\n       \"996  chat.completions.create  gpt-3.5-turbo   \\n\",\n       \"997  chat.completions.create  gpt-3.5-turbo   \\n\",\n       \"998  chat.completions.create  gpt-3.5-turbo   \\n\",\n       \"999  chat.completions.create  gpt-3.5-turbo   \\n\",\n       \"\\n\",\n       \"                                       messages  \\\\\\n\",\n       \"0    [{'role': 'user', 'content': 'say hello'}]   \\n\",\n       \"1    [{'role': 'user', 'content': 'say hello'}]   \\n\",\n       \"2    [{'role': 'user', 'content': 'say hello'}]   \\n\",\n       \"3    [{'role': 'user', 'content': 'say hello'}]   \\n\",\n       \"4    [{'role': 'user', 'content': 'say hello'}]   \\n\",\n       \"..                                          ...   \\n\",\n       \"995  [{'role': 'user', 'content': 'say hello'}]   \\n\",\n       \"996  [{'role': 'user', 'content': 'say hello'}]   \\n\",\n       \"997  [{'role': 'user', 'content': 'say hello'}]   \\n\",\n       \"998  [{'role': 'user', 'content': 'say hello'}]   \\n\",\n       \"999  [{'role': 'user', 'content': 'say hello'}]   \\n\",\n       \"\\n\",\n       \"                                                result  \\n\",\n       \"0    {'id': 'chatcmpl-8qVxz211nsiQQSY2k54r4r141UX83...  \\n\",\n       \"1    {'id': 'chatcmpl-8qVy0TrnGax3XqkDOlEiImosPnvIL...  \\n\",\n       \"2    {'id': 'chatcmpl-8qVxzMdsF5v1je6iAmvjK7BCZKQna...  \\n\",\n       \"3    {'id': 'chatcmpl-8qVy0KiRWYPtQ099p3b1k1HfYBYwT...  \\n\",\n       \"4    {'id': 'chatcmpl-8qVy0M7l6Fg0mvWCpOXomVSqV6Pow...  \\n\",\n       \"..                                                 ...  \\n\",\n       \"995  {'id': 'chatcmpl-8qVyBemEPgb6lV5Opnu8X9UQ7T9iZ...  \\n\",\n       \"996  {'id': 'chatcmpl-8qVyBBlh4PMbI9qtca80UyMbrOGAF...  \\n\",\n       \"997  {'id': 'chatcmpl-8qVyBWveGsMeSLS5SzUbXIiMLvGaS...  \\n\",\n       \"998  {'id': 'chatcmpl-8qVyBbBnX5SleJWSDIHqZ8lS0y15V...  \\n\",\n       \"999  {'id': 'chatcmpl-8qVyBYTizFbBIu3RQ9XlvKDlnSuEG...  \\n\",\n       \"\\n\",\n       \"[1000 rows x 4 columns]\"\n      ]\n     },\n     \"execution_count\": 4,\n     \"metadata\": {},\n     \"output_type\": \"execute_result\"\n    }\n   ],\n   \"source\": [\n    \"from oaib import Auto\\n\",\n    \"\\n\",\n    \"# Automatically set rate limits.\\n\",\n    \"batch = Auto(workers=8)\\n\",\n    \"\\n\",\n    \"# Fetch 1,000 chat completions as quickly as possible, setting rate limits\\n\",\n    \"# automatically from OpenAI's response headers.\\n\",\n    \"for i in range(1000):\\n\",\n    \"    await batch.add(\\n\",\n    \"        \\\"chat.completions.create\\\", \\n\",\n    \"        model=\\\"gpt-3.5-turbo\\\", \\n\",\n    \"        messages=[{\\\"role\\\": \\\"user\\\", \\\"content\\\": \\\"say hello\\\"}]\\n\",\n    \"    )\\n\",\n    \"\\n\",\n    \"# TPM/RPM progress bars will show as full the whole time for Auto runs due to a\\n\",\n    \"# tqdm.notebook bug, but the percentage indicators are correct.\\n\",\n    \"await batch.run()\"\n   ]\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"metadata\": {},\n   \"source\": [\n    \"### Metadata and Index\\n\",\n    \"\\n\",\n    \"You can add custom metadata to your observations with `add(metadata={...}`, and\\n\",\n    \"set the index for the DataFrame with `Batch(index=[...])` and\\n\",\n    \"`Auto(index=[...])`.\"\n   ]\n  },\n  {\n   \"cell_type\": \"code\",\n   \"execution_count\": 5,\n   \"metadata\": {},\n   \"outputs\": [\n    {\n     \"data\": {\n      \"application/vnd.jupyter.widget-view+json\": {\n       \"model_id\": \"6a1c33001c494ac1a8f949d97949c6c6\",\n       \"version_major\": 2,\n       \"version_minor\": 0\n      },\n      \"text/plain\": [\n       \"  0%|          | 0/15 [00:00\u003c?, ?req/s]\"\n      ]\n     },\n     \"metadata\": {},\n     \"output_type\": \"display_data\"\n    },\n    {\n     \"data\": {\n      \"application/vnd.jupyter.widget-view+json\": {\n       \"model_id\": \"fa73772722a4493eae74a26bbeb50102\",\n       \"version_major\": 2,\n       \"version_minor\": 0\n      },\n      \"text/plain\": [\n       \"RPM:   0%|          | 0/1000\"\n      ]\n     },\n     \"metadata\": {},\n     \"output_type\": \"display_data\"\n    },\n    {\n     \"data\": {\n      \"application/vnd.jupyter.widget-view+json\": {\n       \"model_id\": \"45d1aec16e9d4e4c93fa8eedfc8687ef\",\n       \"version_major\": 2,\n       \"version_minor\": 0\n      },\n      \"text/plain\": [\n       \"TPM:   0%|          | 0/10000\"\n      ]\n     },\n     \"metadata\": {},\n     \"output_type\": \"display_data\"\n    },\n    {\n     \"name\": \"stdout\",\n     \"output_type\": \"stream\",\n     \"text\": [\n      \"\\n\",\n      \"Run took 4.74s.\\n\",\n      \"\\n\"\n     ]\n    },\n    {\n     \"data\": {\n      \"text/html\": [\n       \"\u003cdiv\u003e\\n\",\n       \"\u003cstyle scoped\u003e\\n\",\n       \"    .dataframe tbody tr th:only-of-type {\\n\",\n       \"        vertical-align: middle;\\n\",\n       \"    }\\n\",\n       \"\\n\",\n       \"    .dataframe tbody tr th {\\n\",\n       \"        vertical-align: top;\\n\",\n       \"    }\\n\",\n       \"\\n\",\n       \"    .dataframe thead th {\\n\",\n       \"        text-align: right;\\n\",\n       \"    }\\n\",\n       \"\u003c/style\u003e\\n\",\n       \"\u003ctable border=\\\"1\\\" class=\\\"dataframe\\\"\u003e\\n\",\n       \"  \u003cthead\u003e\\n\",\n       \"    \u003ctr style=\\\"text-align: right;\\\"\u003e\\n\",\n       \"      \u003cth\u003e\u003c/th\u003e\\n\",\n       \"      \u003cth\u003e\u003c/th\u003e\\n\",\n       \"      \u003cth\u003eendpoint\u003c/th\u003e\\n\",\n       \"      \u003cth\u003emodel\u003c/th\u003e\\n\",\n       \"      \u003cth\u003emessages\u003c/th\u003e\\n\",\n       \"      \u003cth\u003eresult\u003c/th\u003e\\n\",\n       \"    \u003c/tr\u003e\\n\",\n       \"    \u003ctr\u003e\\n\",\n       \"      \u003cth\u003edifficulty\u003c/th\u003e\\n\",\n       \"      \u003cth\u003ei\u003c/th\u003e\\n\",\n       \"      \u003cth\u003e\u003c/th\u003e\\n\",\n       \"      \u003cth\u003e\u003c/th\u003e\\n\",\n       \"      \u003cth\u003e\u003c/th\u003e\\n\",\n       \"      \u003cth\u003e\u003c/th\u003e\\n\",\n       \"    \u003c/tr\u003e\\n\",\n       \"  \u003c/thead\u003e\\n\",\n       \"  \u003ctbody\u003e\\n\",\n       \"    \u003ctr\u003e\\n\",\n       \"      \u003cth rowspan=\\\"5\\\" valign=\\\"top\\\"\u003eeasy\u003c/th\u003e\\n\",\n       \"      \u003cth\u003e0\u003c/th\u003e\\n\",\n       \"      \u003ctd\u003echat.completions.create\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003egpt-3.5-turbo\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003e[{'role': 'user', 'content': 'difficulty: easy...\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003e{'id': 'chatcmpl-8rdiFkLTnjs4LbX2ZUKyb1UaRcPwH...\u003c/td\u003e\\n\",\n       \"    \u003c/tr\u003e\\n\",\n       \"    \u003ctr\u003e\\n\",\n       \"      \u003cth\u003e1\u003c/th\u003e\\n\",\n       \"      \u003ctd\u003echat.completions.create\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003egpt-3.5-turbo\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003e[{'role': 'user', 'content': 'difficulty: easy...\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003e{'id': 'chatcmpl-8rdiFtEcz6CEvO8K9jZpAdbFaaJ3o...\u003c/td\u003e\\n\",\n       \"    \u003c/tr\u003e\\n\",\n       \"    \u003ctr\u003e\\n\",\n       \"      \u003cth\u003e2\u003c/th\u003e\\n\",\n       \"      \u003ctd\u003echat.completions.create\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003egpt-3.5-turbo\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003e[{'role': 'user', 'content': 'difficulty: easy...\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003e{'id': 'chatcmpl-8rdiFEmy4TfO4iR3aJbh4u9lgAD1W...\u003c/td\u003e\\n\",\n       \"    \u003c/tr\u003e\\n\",\n       \"    \u003ctr\u003e\\n\",\n       \"      \u003cth\u003e3\u003c/th\u003e\\n\",\n       \"      \u003ctd\u003echat.completions.create\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003egpt-3.5-turbo\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003e[{'role': 'user', 'content': 'difficulty: easy...\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003e{'id': 'chatcmpl-8rdiFVw5YuHY8WhuNqJjyJcFO9Mhs...\u003c/td\u003e\\n\",\n       \"    \u003c/tr\u003e\\n\",\n       \"    \u003ctr\u003e\\n\",\n       \"      \u003cth\u003e4\u003c/th\u003e\\n\",\n       \"      \u003ctd\u003echat.completions.create\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003egpt-3.5-turbo\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003e[{'role': 'user', 'content': 'difficulty: easy...\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003e{'id': 'chatcmpl-8rdiGoYqWA5wH3xoFKVl8pRGGTuDZ...\u003c/td\u003e\\n\",\n       \"    \u003c/tr\u003e\\n\",\n       \"    \u003ctr\u003e\\n\",\n       \"      \u003cth rowspan=\\\"5\\\" valign=\\\"top\\\"\u003ehard\u003c/th\u003e\\n\",\n       \"      \u003cth\u003e0\u003c/th\u003e\\n\",\n       \"      \u003ctd\u003echat.completions.create\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003egpt-3.5-turbo\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003e[{'role': 'user', 'content': 'difficulty: hard...\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003e{'id': 'chatcmpl-8rdiG8ZAUhcsBOtgtPuOjblzEo14a...\u003c/td\u003e\\n\",\n       \"    \u003c/tr\u003e\\n\",\n       \"    \u003ctr\u003e\\n\",\n       \"      \u003cth\u003e1\u003c/th\u003e\\n\",\n       \"      \u003ctd\u003echat.completions.create\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003egpt-3.5-turbo\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003e[{'role': 'user', 'content': 'difficulty: hard...\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003e{'id': 'chatcmpl-8rdiGgl9uEe4ASQgt5uzwMpeJhnU6...\u003c/td\u003e\\n\",\n       \"    \u003c/tr\u003e\\n\",\n       \"    \u003ctr\u003e\\n\",\n       \"      \u003cth\u003e2\u003c/th\u003e\\n\",\n       \"      \u003ctd\u003echat.completions.create\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003egpt-3.5-turbo\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003e[{'role': 'user', 'content': 'difficulty: hard...\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003e{'id': 'chatcmpl-8rdiGfVkiZqrE1p2TxeKnCxc2zzUb...\u003c/td\u003e\\n\",\n       \"    \u003c/tr\u003e\\n\",\n       \"    \u003ctr\u003e\\n\",\n       \"      \u003cth\u003e3\u003c/th\u003e\\n\",\n       \"      \u003ctd\u003echat.completions.create\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003egpt-3.5-turbo\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003e[{'role': 'user', 'content': 'difficulty: hard...\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003e{'id': 'chatcmpl-8rdiGdKxmYC6mS4QSuiW3HjsMRW05...\u003c/td\u003e\\n\",\n       \"    \u003c/tr\u003e\\n\",\n       \"    \u003ctr\u003e\\n\",\n       \"      \u003cth\u003e4\u003c/th\u003e\\n\",\n       \"      \u003ctd\u003echat.completions.create\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003egpt-3.5-turbo\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003e[{'role': 'user', 'content': 'difficulty: hard...\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003e{'id': 'chatcmpl-8rdiGqTe3MVy8FGJ6qtVAjxmMODy6...\u003c/td\u003e\\n\",\n       \"    \u003c/tr\u003e\\n\",\n       \"    \u003ctr\u003e\\n\",\n       \"      \u003cth rowspan=\\\"5\\\" valign=\\\"top\\\"\u003emedium\u003c/th\u003e\\n\",\n       \"      \u003cth\u003e0\u003c/th\u003e\\n\",\n       \"      \u003ctd\u003echat.completions.create\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003egpt-3.5-turbo\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003e[{'role': 'user', 'content': 'difficulty: medi...\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003e{'id': 'chatcmpl-8rdiGsk7ohvIVwzuFMfO3cH9zNLW4...\u003c/td\u003e\\n\",\n       \"    \u003c/tr\u003e\\n\",\n       \"    \u003ctr\u003e\\n\",\n       \"      \u003cth\u003e1\u003c/th\u003e\\n\",\n       \"      \u003ctd\u003echat.completions.create\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003egpt-3.5-turbo\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003e[{'role': 'user', 'content': 'difficulty: medi...\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003e{'id': 'chatcmpl-8rdiG64Y66W8CZZ9MI4xiVNHheHiF...\u003c/td\u003e\\n\",\n       \"    \u003c/tr\u003e\\n\",\n       \"    \u003ctr\u003e\\n\",\n       \"      \u003cth\u003e2\u003c/th\u003e\\n\",\n       \"      \u003ctd\u003echat.completions.create\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003egpt-3.5-turbo\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003e[{'role': 'user', 'content': 'difficulty: medi...\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003e{'id': 'chatcmpl-8rdiGGmXfXx0uRuKafeKODGQe42vz...\u003c/td\u003e\\n\",\n       \"    \u003c/tr\u003e\\n\",\n       \"    \u003ctr\u003e\\n\",\n       \"      \u003cth\u003e3\u003c/th\u003e\\n\",\n       \"      \u003ctd\u003echat.completions.create\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003egpt-3.5-turbo\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003e[{'role': 'user', 'content': 'difficulty: medi...\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003e{'id': 'chatcmpl-8rdiG95XCozUiFXY7ryA4rSfzbwRk...\u003c/td\u003e\\n\",\n       \"    \u003c/tr\u003e\\n\",\n       \"    \u003ctr\u003e\\n\",\n       \"      \u003cth\u003e4\u003c/th\u003e\\n\",\n       \"      \u003ctd\u003echat.completions.create\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003egpt-3.5-turbo\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003e[{'role': 'user', 'content': 'difficulty: medi...\u003c/td\u003e\\n\",\n       \"      \u003ctd\u003e{'id': 'chatcmpl-8rdiGi6nF5FLMdPIYrkrYgI15Yfcw...\u003c/td\u003e\\n\",\n       \"    \u003c/tr\u003e\\n\",\n       \"  \u003c/tbody\u003e\\n\",\n       \"\u003c/table\u003e\\n\",\n       \"\u003c/div\u003e\"\n      ],\n      \"text/plain\": [\n       \"                             endpoint          model  \\\\\\n\",\n       \"difficulty i                                           \\n\",\n       \"easy       0  chat.completions.create  gpt-3.5-turbo   \\n\",\n       \"           1  chat.completions.create  gpt-3.5-turbo   \\n\",\n       \"           2  chat.completions.create  gpt-3.5-turbo   \\n\",\n       \"           3  chat.completions.create  gpt-3.5-turbo   \\n\",\n       \"           4  chat.completions.create  gpt-3.5-turbo   \\n\",\n       \"hard       0  chat.completions.create  gpt-3.5-turbo   \\n\",\n       \"           1  chat.completions.create  gpt-3.5-turbo   \\n\",\n       \"           2  chat.completions.create  gpt-3.5-turbo   \\n\",\n       \"           3  chat.completions.create  gpt-3.5-turbo   \\n\",\n       \"           4  chat.completions.create  gpt-3.5-turbo   \\n\",\n       \"medium     0  chat.completions.create  gpt-3.5-turbo   \\n\",\n       \"           1  chat.completions.create  gpt-3.5-turbo   \\n\",\n       \"           2  chat.completions.create  gpt-3.5-turbo   \\n\",\n       \"           3  chat.completions.create  gpt-3.5-turbo   \\n\",\n       \"           4  chat.completions.create  gpt-3.5-turbo   \\n\",\n       \"\\n\",\n       \"                                                       messages  \\\\\\n\",\n       \"difficulty i                                                      \\n\",\n       \"easy       0  [{'role': 'user', 'content': 'difficulty: easy...   \\n\",\n       \"           1  [{'role': 'user', 'content': 'difficulty: easy...   \\n\",\n       \"           2  [{'role': 'user', 'content': 'difficulty: easy...   \\n\",\n       \"           3  [{'role': 'user', 'content': 'difficulty: easy...   \\n\",\n       \"           4  [{'role': 'user', 'content': 'difficulty: easy...   \\n\",\n       \"hard       0  [{'role': 'user', 'content': 'difficulty: hard...   \\n\",\n       \"           1  [{'role': 'user', 'content': 'difficulty: hard...   \\n\",\n       \"           2  [{'role': 'user', 'content': 'difficulty: hard...   \\n\",\n       \"           3  [{'role': 'user', 'content': 'difficulty: hard...   \\n\",\n       \"           4  [{'role': 'user', 'content': 'difficulty: hard...   \\n\",\n       \"medium     0  [{'role': 'user', 'content': 'difficulty: medi...   \\n\",\n       \"           1  [{'role': 'user', 'content': 'difficulty: medi...   \\n\",\n       \"           2  [{'role': 'user', 'content': 'difficulty: medi...   \\n\",\n       \"           3  [{'role': 'user', 'content': 'difficulty: medi...   \\n\",\n       \"           4  [{'role': 'user', 'content': 'difficulty: medi...   \\n\",\n       \"\\n\",\n       \"                                                         result  \\n\",\n       \"difficulty i                                                     \\n\",\n       \"easy       0  {'id': 'chatcmpl-8rdiFkLTnjs4LbX2ZUKyb1UaRcPwH...  \\n\",\n       \"           1  {'id': 'chatcmpl-8rdiFtEcz6CEvO8K9jZpAdbFaaJ3o...  \\n\",\n       \"           2  {'id': 'chatcmpl-8rdiFEmy4TfO4iR3aJbh4u9lgAD1W...  \\n\",\n       \"           3  {'id': 'chatcmpl-8rdiFVw5YuHY8WhuNqJjyJcFO9Mhs...  \\n\",\n       \"           4  {'id': 'chatcmpl-8rdiGoYqWA5wH3xoFKVl8pRGGTuDZ...  \\n\",\n       \"hard       0  {'id': 'chatcmpl-8rdiG8ZAUhcsBOtgtPuOjblzEo14a...  \\n\",\n       \"           1  {'id': 'chatcmpl-8rdiGgl9uEe4ASQgt5uzwMpeJhnU6...  \\n\",\n       \"           2  {'id': 'chatcmpl-8rdiGfVkiZqrE1p2TxeKnCxc2zzUb...  \\n\",\n       \"           3  {'id': 'chatcmpl-8rdiGdKxmYC6mS4QSuiW3HjsMRW05...  \\n\",\n       \"           4  {'id': 'chatcmpl-8rdiGqTe3MVy8FGJ6qtVAjxmMODy6...  \\n\",\n       \"medium     0  {'id': 'chatcmpl-8rdiGsk7ohvIVwzuFMfO3cH9zNLW4...  \\n\",\n       \"           1  {'id': 'chatcmpl-8rdiG64Y66W8CZZ9MI4xiVNHheHiF...  \\n\",\n       \"           2  {'id': 'chatcmpl-8rdiGGmXfXx0uRuKafeKODGQe42vz...  \\n\",\n       \"           3  {'id': 'chatcmpl-8rdiG95XCozUiFXY7ryA4rSfzbwRk...  \\n\",\n       \"           4  {'id': 'chatcmpl-8rdiGi6nF5FLMdPIYrkrYgI15Yfcw...  \"\n      ]\n     },\n     \"execution_count\": 5,\n     \"metadata\": {},\n     \"output_type\": \"execute_result\"\n    }\n   ],\n   \"source\": [\n    \"from oaib import Batch\\n\",\n    \"\\n\",\n    \"n = 5\\n\",\n    \"batch = Batch(rpm=1000, tpm=10000, workers=5, index=[\\\"difficulty\\\", \\\"i\\\"])\\n\",\n    \"difficulties = [\\\"easy\\\", \\\"medium\\\", \\\"hard\\\"]\\n\",\n    \"\\n\",\n    \"for difficulty in difficulties:\\n\",\n    \"    for i in range(n):\\n\",\n    \"        await batch.add(\\n\",\n    \"            \\\"chat.completions.create\\\",\\n\",\n    \"            metadata={\\\"difficulty\\\": difficulty, \\\"i\\\": i},\\n\",\n    \"            model=\\\"gpt-3.5-turbo\\\",\\n\",\n    \"            messages=[{\\n\",\n    \"                \\\"role\\\": \\\"user\\\", \\n\",\n    \"                \\\"content\\\": f\\\"difficulty: {difficulty}. write a math problem.\\\"\\n\",\n    \"            }]\\n\",\n    \"        )\\n\",\n    \"\\n\",\n    \"await batch.run()\"\n   ]\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"metadata\": {},\n   \"source\": [\n    \"\\n\",\n    \"## Notes\\n\",\n    \"\\n\",\n    \"1. It is not possible to perfectly guarantee the tokens per minute limit is not\\n\",\n    \"   breached because we cannot know the total token usage until the response\\n\",\n    \"   comes back.\\n\",\n    \"   \\n\",\n    \"   Use the `safety` param to set the rate limit tolerance.  By default it is set\\n\",\n    \"   to 10%, and will wait until the predicted TPM (the current TPM plus the\\n\",\n    \"   average number of tokens per request) drops below 90% of the limit.\\n\",\n    \"\\n\",\n    \"  \\n\",\n    \"2. By default, important logs are stored at `oaib.txt`.  This can be disabled\\n\",\n    \"   using `loglevel=0`.\\n\",\n    \"\\n\",\n    \"3. There's an error with TPM/RPM progress bar display in Jupyter Notebooks for\\n\",\n    \"   the `Auto` class only. This is caused by a `tqdm.notebook` bug where only the\\n\",\n    \"   initial totals (here, our limits) are used to calculate the width of the bar,\\n\",\n    \"   and the `Auto` class updates these values only after the first request. The\\n\",\n    \"   text percentage displays are accurate.\"\n   ]\n  }\n ],\n \"metadata\": {\n  \"kernelspec\": {\n   \"display_name\": \"Python 3\",\n   \"language\": \"python\",\n   \"name\": \"python3\"\n  },\n  \"language_info\": {\n   \"codemirror_mode\": {\n    \"name\": \"ipython\",\n    \"version\": 3\n   },\n   \"file_extension\": \".py\",\n   \"mimetype\": \"text/x-python\",\n   \"name\": \"python\",\n   \"nbconvert_exporter\": \"python\",\n   \"pygments_lexer\": \"ipython3\",\n   \"version\": \"3.0.0\"\n  }\n },\n \"nbformat\": 4,\n \"nbformat_minor\": 2\n}\n","project_url":"https://awesome.ecosyste.ms/api/v1/projects/github.com%2Fspellcraftai%2Foaib","html_url":"https://awesome.ecosyste.ms/projects/github.com%2Fspellcraftai%2Foaib","lists_url":"https://awesome.ecosyste.ms/api/v1/projects/github.com%2Fspellcraftai%2Foaib/lists"}