{"id":18283673,"url":"https://github.com/hhsecond/lightning-triton","last_synced_at":"2025-07-20T20:33:09.294Z","repository":{"id":94514309,"uuid":"563279081","full_name":"hhsecond/lightning-triton","owner":"hhsecond","description":"Triton serve from Lightning with Stable Diffusion","archived":false,"fork":false,"pushed_at":"2022-11-09T11:48:57.000Z","size":578,"stargazers_count":2,"open_issues_count":0,"forks_count":0,"subscribers_count":1,"default_branch":"main","last_synced_at":"2025-04-09T05:43:23.538Z","etag":null,"topics":[],"latest_commit_sha":null,"homepage":null,"language":"Python","has_issues":true,"has_wiki":null,"has_pages":null,"mirror_url":null,"source_name":null,"license":"apache-2.0","status":null,"scm":"git","pull_requests_enabled":true,"icon_url":"https://github.com/hhsecond.png","metadata":{"files":{"readme":"README.md","changelog":null,"contributing":null,"funding":null,"license":"LICENSE","code_of_conduct":null,"threat_model":null,"audit":null,"citation":null,"codeowners":null,"security":null,"support":null,"governance":null,"roadmap":null,"authors":null,"dei":null,"publiccode":null,"codemeta":null}},"created_at":"2022-11-08T09:31:07.000Z","updated_at":"2024-02-19T20:45:13.000Z","dependencies_parsed_at":null,"dependency_job_id":"b7fae2e1-3fd2-4b43-913b-b8ec179861e2","html_url":"https://github.com/hhsecond/lightning-triton","commit_stats":null,"previous_names":[],"tags_count":0,"template":false,"template_full_name":null,"purl":"pkg:github/hhsecond/lightning-triton","repository_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/hhsecond%2Flightning-triton","tags_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/hhsecond%2Flightning-triton/tags","releases_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/hhsecond%2Flightning-triton/releases","manifests_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/hhsecond%2Flightning-triton/manifests","owner_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners/hhsecond","download_url":"https://codeload.github.com/hhsecond/lightning-triton/tar.gz/refs/heads/main","sbom_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/hhsecond%2Flightning-triton/sbom","host":{"name":"GitHub","url":"https://github.com","kind":"github","repositories_count":266194233,"owners_count":23890957,"icon_url":"https://github.com/github.png","version":null,"created_at":"2022-05-30T11:31:42.601Z","updated_at":"2022-07-04T15:15:14.044Z","host_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub","repositories_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories","repository_names_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repository_names","owners_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners"}},"keywords":[],"created_at":"2024-11-05T13:10:20.534Z","updated_at":"2025-07-20T20:33:09.273Z","avatar_url":"https://github.com/hhsecond.png","language":"Python","funding_links":[],"categories":[],"sub_categories":[],"readme":"# lightning-triton\nTriton serve from Lightning with Stable Diffusion\n\n# Run the app\n\nCopy the code below into an `app.py` file and run below command to run it on cloud \n\n```bash\nlightning run app app.py --cloud\n```\n\n```python\n# git clone https://github.com/hhsecond/lightning-triton.git\n# mv lightning-triton/* .\n# pip install -r triton-requirements.txt\n\nimport base64\nimport subprocess\nimport tarfile\nfrom io import BytesIO\nfrom typing import List\nfrom urllib import request\n\nimport lightning as L\nimport uvicorn\nfrom fastapi import FastAPI\nfrom lightning_api_access import APIAccessFrontend\nfrom pydantic import BaseModel\n\n\nclass DiffusionBuildConfig(L.BuildConfig):\n    def build_commands(self) -\u003e List[str]:\n        return [\"pip install -r triton-requirements.txt\"]\n\n\nclass TritonServe(L.LightningWork):\n    def __init__(self, cloud_compute: L.CloudCompute):\n        super().__init__(\n            cloud_compute=cloud_compute,\n            cloud_build_config=DiffusionBuildConfig(\n                image=\"ghcr.io/gridai/lightning-triton:v0.9\"\n            ),\n            parallel=True,\n        )\n\n    def run(self):\n        request.urlretrieve(\n            \"https://pl-public-data.s3.amazonaws.com/dream_stable_diffusion/sd_weights.tar.gz\",\n            \"sd_weights.tar.gz\",\n        )\n        file = tarfile.open(\"sd_weights.tar.gz\")\n        file.extractall(\"model_repository/diffusion/1/stable_diffusion_weights\")\n        subprocess.run(\n            [\n                \"tritonserver\",\n                \"--model-repository=model_repository\",\n                \"--http-port\",\n                str(self.port),\n            ]\n        )\n\n\nclass APIComponent(L.LightningWork):\n    def __init__(self):\n        super().__init__(cloud_build_config=DiffusionBuildConfig(), parallel=True)\n\n    def run(self, serve_engine_url):\n        import numpy as np\n        import tritonclient.http as httpclient\n        from PIL import Image\n        from tritonclient.utils import np_to_triton_dtype\n\n        fastapi_app = FastAPI()\n\n        class Data(BaseModel):\n            prompt: str\n\n        @fastapi_app.post(\"/predict\")\n        async def predict(data: Data):\n            client = httpclient.InferenceServerClient(\n                url=serve_engine_url, connection_timeout=1200.0, network_timeout=1200.0\n            )\n            text_obj = np.array([data.prompt], dtype=\"object\").reshape((-1, 1))\n            input_text = httpclient.InferInput(\n                \"prompt\", text_obj.shape, np_to_triton_dtype(text_obj.dtype)\n            )\n            input_text.set_data_from_numpy(text_obj)\n            output_img = httpclient.InferRequestedOutput(\"generated_image\")\n            query_response = client.infer(\n                model_name=\"diffusion\",\n                inputs=[input_text],\n                outputs=[output_img],\n                timeout=240,\n            )\n            image = Image.fromarray(\n                np.squeeze(query_response.as_numpy(\"generated_image\").astype(np.uint8))\n            )\n            buffered = BytesIO()\n            image.save(buffered, format=\"PNG\")\n            img_str = base64.b64encode(buffered.getvalue()).decode(\"utf-8\")\n            return {\"image\": f\"data:image/png;base64,{img_str}\"}\n\n        uvicorn.run(\n            fastapi_app,\n            host=self.host,\n            port=self.port,\n            loop=\"uvloop\",\n            timeout_keep_alive=60,\n        )\n\n\nclass APIUsageFlow(L.LightningFlow):\n    def __init__(self, api_url: str = \"\"):\n        super().__init__()\n        self.api_url = api_url\n\n    def configure_layout(self):\n        return APIAccessFrontend(\n            apis=[\n                {\n                    \"name\": \"Generate Image\",\n                    \"url\": f\"{self.api_url}/predict\",\n                    \"method\": \"POST\",\n                    \"request\": {\"prompt\": \"cats in hats\"},\n                    \"response\": {\n                        \"image\": \"data:image/png;base64,\u003cimage-actual-content\u003e\"\n                    },\n                }\n            ]\n        )\n\n\nclass DiffusionServeFlow(L.LightningFlow):\n    def __init__(self):\n        super().__init__()\n        self.triton_server = TritonServe(\n            cloud_compute=L.CloudCompute(\"cpu-medium\", disk_size=100)\n        )\n        self.api_component = APIComponent()\n        self.api_usage_flow = APIUsageFlow()\n\n    def run(self):\n        self.triton_server.run()\n        if self.triton_server.is_running and self.triton_server.internal_ip:\n            self.api_component.run(\n                serve_engine_url=f\"{self.triton_server.internal_ip}:{self.triton_server.port}\"\n            )\n        if self.api_component.is_running and self.api_component.url:\n            self.api_usage_flow.api_url = self.api_component.url\n\n    def configure_layout(self):\n        return [\n            {\"name\": \"API\", \"content\": self.api_usage_flow}\n        ]\n\n\napp = L.LightningApp(DiffusionServeFlow())\n```","project_url":"https://awesome.ecosyste.ms/api/v1/projects/github.com%2Fhhsecond%2Flightning-triton","html_url":"https://awesome.ecosyste.ms/projects/github.com%2Fhhsecond%2Flightning-triton","lists_url":"https://awesome.ecosyste.ms/api/v1/projects/github.com%2Fhhsecond%2Flightning-triton/lists"}