{"id":13601588,"url":"https://github.com/neuralmagic/deepsparse","last_synced_at":"2025-05-14T01:11:48.036Z","repository":{"id":37058551,"uuid":"321426168","full_name":"neuralmagic/deepsparse","owner":"neuralmagic","description":"Sparsity-aware deep learning inference runtime for CPUs","archived":false,"fork":false,"pushed_at":"2025-05-10T15:36:02.000Z","size":143783,"stargazers_count":3148,"open_issues_count":1,"forks_count":185,"subscribers_count":55,"default_branch":"main","last_synced_at":"2025-05-13T14:17:30.045Z","etag":null,"topics":["computer-vision","cpus","deepsparse","inference","llm-inference","machinelearning","nlp","object-detection","onnx","performance","pretrained-models","pruning","quantization","sparsification"],"latest_commit_sha":null,"homepage":"https://neuralmagic.com/deepsparse/","language":"Python","has_issues":true,"has_wiki":null,"has_pages":null,"mirror_url":null,"source_name":null,"license":"other","status":null,"scm":"git","pull_requests_enabled":true,"icon_url":"https://github.com/neuralmagic.png","metadata":{"files":{"readme":"README.md","changelog":null,"contributing":"CONTRIBUTING.md","funding":null,"license":"LICENSE","code_of_conduct":"CODE_OF_CONDUCT.md","threat_model":null,"audit":null,"citation":"CITATION.cff","codeowners":null,"security":null,"support":null,"governance":null,"roadmap":null,"authors":null,"dei":null,"publiccode":null,"codemeta":null,"zenodo":null}},"created_at":"2020-12-14T17:40:38.000Z","updated_at":"2025-05-13T09:40:29.000Z","dependencies_parsed_at":"2023-12-18T21:55:14.901Z","dependency_job_id":"f85484dd-8590-4aa6-86b6-4e10140fe024","html_url":"https://github.com/neuralmagic/deepsparse","commit_stats":{"total_commits":1052,"total_committers":49,"mean_commits":21.46938775510204,"dds":0.8212927756653993,"last_synced_commit":"36b92eeb730a74a787cea467c9132eaa1b78167f"},"previous_names":[],"tags_count":41,"template":false,"template_full_name":null,"repository_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/neuralmagic%2Fdeepsparse","tags_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/neuralmagic%2Fdeepsparse/tags","releases_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/neuralmagic%2Fdeepsparse/releases","manifests_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/neuralmagic%2Fdeepsparse/manifests","owner_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners/neuralmagic","download_url":"https://codeload.github.com/neuralmagic/deepsparse/tar.gz/refs/heads/main","host":{"name":"GitHub","url":"https://github.com","kind":"github","repositories_count":254049501,"owners_count":22006077,"icon_url":"https://github.com/github.png","version":null,"created_at":"2022-05-30T11:31:42.601Z","updated_at":"2022-07-04T15:15:14.044Z","host_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub","repositories_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories","repository_names_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repository_names","owners_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners"}},"keywords":["computer-vision","cpus","deepsparse","inference","llm-inference","machinelearning","nlp","object-detection","onnx","performance","pretrained-models","pruning","quantization","sparsification"],"created_at":"2024-08-01T18:01:04.989Z","updated_at":"2025-05-14T01:11:43.020Z","avatar_url":"https://github.com/neuralmagic.png","language":"Python","funding_links":[],"categories":["HarmonyOS","Open Source Projects","Python","Deployment and Serving","A01_文本生成_文本对话","Repos","🌐 **Edge Deployment Frameworks**"],"sub_categories":["Windows Manager","大语言对话模型及数据","💎 **DeepSparse \u0026 SparseML** - Neural Magic"],"readme":"\u003c!--\nCopyright (c) 2021 - present / Neuralmagic, Inc. All Rights Reserved.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n   http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing,\nsoftware distributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n--\u003e\n\n\n\u003cdiv style=\"display: flex; flex-direction: column; align-items: center;\"\u003e\n  \u003ch1 style=\"display: flex; align-items: center;\" \u003e\n     \u003cimg width=\"60\" height=\"60\" alt=\"tool icon\" src=\"https://neuralmagic.com/wp-content/uploads/2024/03/icon_DeepSparse-005.png\" /\u003e\n      \u003cspan\u003e\u0026nbsp;\u0026nbsp;DeepSparse\u003c/span\u003e\n  \u003c/h1\u003e\n  \u003ch4\u003eSparsity-aware deep learning inference runtime for CPUs\u003c/h4\u003e\n  \u003cdiv align=\"center\"\u003e\n  \u003ca href=\"https://docs.neuralmagic.com/deepsparse/\"\u003e\n    \u003cimg alt=\"Documentation\" src=\"https://img.shields.io/badge/documentation-darkred?\u0026style=for-the-badge\u0026logo=read-the-docs\" height=\"20\" /\u003e\n  \u003c/a\u003e\n  \u003ca href=\"https://neuralmagic.com/community/\"\u003e\n    \u003cimg alt=\"Slack\" src=\"https://img.shields.io/badge/slack-purple?style=for-the-badge\u0026logo=slack\" height=\"20\" /\u003e\n  \u003c/a\u003e\n  \u003ca href=\"https://github.com/neuralmagic/deepsparse/issues/\"\u003e\n    \u003cimg alt=\"Support\" src=\"https://img.shields.io/badge/support%20forums-navy?style=for-the-badge\u0026logo=github\" height=\"20\" /\u003e\n  \u003c/a\u003e\n  \u003ca href=\"https://www.youtube.com/channel/UCo8dO_WMGYbWCRnj_Dxr4EA\"\u003e\n    \u003cimg alt=\"YouTube\" src=\"https://img.shields.io/badge/-YouTube-red?\u0026style=for-the-badge\u0026logo=youtube\u0026logoColor=white\" height=\"20\" /\u003e\n  \u003c/a\u003e\n  \u003ca href=\"https://twitter.com/neuralmagic\"\u003e\n    \u003cimg alt=\"Twitter\" src=\"https://img.shields.io/twitter/follow/neuralmagic?color=darkgreen\u0026label=Follow\u0026style=social\" height=\"20\" /\u003e\n  \u003c/a\u003e\n\u003c/div\u003e\n\n[DeepSparse](https://github.com/neuralmagic/deepsparse) is a CPU inference runtime that takes advantage of sparsity to accelerate neural network inference. Coupled with [SparseML](https://github.com/neuralmagic/sparseml), our optimization library for pruning and quantizing your models, DeepSparse delivers exceptional inference performance on CPU hardware.\n\n\u003cp align=\"center\"\u003e\n   \u003cimg alt=\"NM Flow\" src=\"https://github.com/neuralmagic/deepsparse/assets/3195154/51e62fe7-9d9a-4fa5-a774-877158da1e29\" width=\"60%\" /\u003e\n\u003c/p\u003e\n\n## ✨NEW✨ DeepSparse LLMs\n\nNeural Magic is excited to announce initial support for performant LLM inference in DeepSparse with:\n- sparse kernels for speedups and memory savings from unstructured sparse weights.\n- 8-bit weight and activation quantization support.\n- efficient usage of cached attention keys and values for minimal memory movement.\n\n![mpt-chat-comparison](https://github.com/neuralmagic/deepsparse/assets/3195154/ccf39323-4603-4489-8462-7b103872aeb3)\n\n### Try It Now\n\nInstall (requires Linux):\n```bash\npip install -U deepsparse-nightly[llm]\n```\n\nRun inference:\n```python\nfrom deepsparse import TextGeneration\npipeline = TextGeneration(model=\"zoo:mpt-7b-dolly_mpt_pretrain-pruned50_quantized\")\n\nprompt=\"\"\"\nBelow is an instruction that describes a task. Write a response that appropriately completes the request. ### Instruction: what is sparsity? ### Response:\n\"\"\"\nprint(pipeline(prompt, max_new_tokens=75).generations[0].text)\n\n# Sparsity is the property of a matrix or other data structure in which a large number of elements are zero and a smaller number of elements are non-zero. In the context of machine learning, sparsity can be used to improve the efficiency of training and prediction.\n```\n\nCheck out the [`TextGeneration` documentation for usage details](https://github.com/neuralmagic/deepsparse/blob/main/docs/llms/text-generation-pipeline.md) and get the [latest sparsified LLMs on our HF Collection](https://huggingface.co/collections/neuralmagic/deepsparse-sparse-llms-659d61e81774dd48343642bf).\n\n### Sparsity :handshake: Performance\n\nDeveloped in collaboration with IST Austria, [our recent paper](https://arxiv.org/abs/2310.06927) details a new technique called **Sparse Fine-Tuning**, which allows us to prune MPT-7B to 60% sparsity during fine-tuning without drop in accuracy. With our new support for LLMs, DeepSparse accelerates the sparse-quantized model 7x over the dense baseline:\n\n\u003cdiv align=\"center\"\u003e\n    \u003cimg src=\"https://github.com/neuralmagic/deepsparse/assets/3195154/8687401c-f479-4999-ba6b-e01c747dace9\" width=\"60%\"/\u003e\n\u003c/div\u003e\n\n\u003e [Learn more about our Sparse Fine-Tuning research.](https://github.com/neuralmagic/deepsparse/blob/main/research/mpt#sparse-finetuned-llms-with-deepsparse)\n\n\u003e [Check out the model running live on Hugging Face.](https://huggingface.co/spaces/neuralmagic/sparse-mpt-7b-gsm8k)\n\n### LLM Roadmap\n\nFollowing this initial launch, we are rapidly expanding our support for LLMs, including:\n\n1. Productizing Sparse Fine-Tuning: Enable external users to apply sparse fine-tuning to their datasets via SparseML.\n2. Expanding model support: Apply our sparse fine-tuning results to Llama 2 and Mistral models.\n3. Pushing for higher sparsity: Improving our pruning algorithms to reach even higher sparsity.\n\n## Computer Vision and NLP Models\n\nIn addition to LLMs, DeepSparse supports many variants of CNNs and Transformer models, such as BERT, ViT, ResNet, EfficientNet, YOLOv5/8, and many more! Take a look at the [Computer Vision](https://sparsezoo.neuralmagic.com/?modelSet=computer_vision) and [Natural Language Processing](https://sparsezoo.neuralmagic.com/?modelSet=natural_language_processing) domains of [SparseZoo](https://sparsezoo.neuralmagic.com/), our home for optimized models.\n\n### Installation\n\nInstall via [PyPI](https://pypi.org/project/deepsparse/) ([optional dependencies detailed here](https://github.com/neuralmagic/deepsparse/tree/main/docs/user-guide/installation.md)):\n\n```bash\npip install deepsparse \n```\n\nTo experiment with the latest features, there is a nightly build available using `pip install deepsparse-nightly` or you can clone and install from source using `pip install -e path/to/deepsparse`.\n\n#### System Requirements\n- Hardware: [x86 AVX2, AVX-512, AVX-512 VNNI and ARM v8.2+](https://github.com/neuralmagic/deepsparse/tree/main/docs/user-guide/hardware-support.md)\n- Operating System: Linux\n- Python: 3.8-3.11\n- ONNX versions 1.5.0-1.15.0, ONNX opset version 11 or higher\n\nFor those using Mac or Windows, we recommend using Linux containers with Docker.\n\n## Deployment APIs\n\nDeepSparse includes three deployment APIs:\n\n- **Engine** is the lowest-level API. With Engine, you compile an ONNX model, pass tensors as input, and receive the raw outputs.\n- **Pipeline** wraps the Engine with pre- and post-processing. With Pipeline, you pass raw data and receive the prediction.\n- **Server** wraps Pipelines with a REST API using FastAPI. With Server, you send raw data over HTTP and receive the prediction.\n\n### Engine\n\nThe example below downloads a 90% pruned-quantized BERT model for sentiment analysis in ONNX format from SparseZoo, compiles the model, and runs inference on randomly generated input. Users can provide their own ONNX models, whether dense or sparse.\n\n```python\nfrom deepsparse import Engine\n\n# download onnx, compile\nzoo_stub = \"zoo:nlp/sentiment_analysis/obert-base/pytorch/huggingface/sst2/pruned90_quant-none\"\ncompiled_model = Engine(model=zoo_stub, batch_size=1)\n\n# run inference (input is raw numpy tensors, output is raw scores)\ninputs = compiled_model.generate_random_inputs()\noutput = compiled_model(inputs)\nprint(output)\n\n# \u003e [array([[-0.3380675 ,  0.09602544]], dtype=float32)] \u003c\u003c raw scores\n```\n\n### Pipeline\n\nPipelines wrap Engine with pre- and post-processing, enabling you to pass raw data and receive the post-processed prediction. The example below downloads a 90% pruned-quantized BERT model for sentiment analysis in ONNX format from SparseZoo, sets up a pipeline, and runs inference on sample data.\n\n```python\nfrom deepsparse import Pipeline\n\n# download onnx, set up pipeline\nzoo_stub = \"zoo:nlp/sentiment_analysis/obert-base/pytorch/huggingface/sst2/pruned90_quant-none\"  \nsentiment_analysis_pipeline = Pipeline.create(\n  task=\"sentiment-analysis\",    # name of the task\n  model_path=zoo_stub,          # zoo stub or path to local onnx file\n)\n\n# run inference (input is a sentence, output is the prediction)\nprediction = sentiment_analysis_pipeline(\"I love using DeepSparse Pipelines\")\nprint(prediction)\n# \u003e labels=['positive'] scores=[0.9954759478569031]\n```\n\n### Server\n\nServer wraps Pipelines with REST APIs, enabling you to set up a model-serving endpoint running DeepSparse. This enables you to send raw data to DeepSparse over HTTP and receive the post-processed predictions. DeepSparse Server is launched from the command line and configured via arguments or a server configuration file. The following downloads a 90% pruned-quantized BERT model for sentiment analysis in ONNX format from SparseZoo and launches a sentiment analysis endpoint:\n\n```bash\ndeepsparse.server \\\n  --task sentiment-analysis \\\n  --model_path zoo:nlp/sentiment_analysis/obert-base/pytorch/huggingface/sst2/pruned90_quant-none\n```\n\nSending a request:\n\n```python\nimport requests\n\nurl = \"http://localhost:5543/v2/models/sentiment_analysis/infer\" # Server's port default to 5543\nobj = {\"sequences\": \"Snorlax loves my Tesla!\"}\n\nresponse = requests.post(url, json=obj)\nprint(response.text)\n# {\"labels\":[\"positive\"],\"scores\":[0.9965094327926636]}\n```\n\n### Additional Resources\n- [Use Cases Page](https://github.com/neuralmagic/deepsparse/tree/main/docs/use-cases) for more details on supported tasks\n- [Pipelines User Guide](https://github.com/neuralmagic/deepsparse/tree/main/docs/user-guide/deepsparse-pipelines.md) for Pipeline documentation\n- [Server User Guide](https://github.com/neuralmagic/deepsparse/tree/main/docs/user-guide/deepsparse-server.md) for Server documentation\n- [Benchmarking User Guide](https://github.com/neuralmagic/deepsparse/tree/main/docs/user-guide/deepsparse-benchmarking.md) for benchmarking documentation\n- [Cloud Deployments and Demos](https://github.com/neuralmagic/deepsparse/tree/main/examples/)\n- [User Guide](https://github.com/neuralmagic/deepsparse/tree/main/docs/user-guide) for more detailed documentation\n\n\n## Product Usage Analytics\n\nDeepSparse gathers basic usage telemetry, including, but not limited to, Invocations, Package, Version, and IP Address, for Product Usage Analytics purposes. Review Neural Magic's [Products Privacy Policy](https://neuralmagic.com/legal/) for further details on how we process this data. \n\nTo disable Product Usage Analytics, run:\n```bash\nexport NM_DISABLE_ANALYTICS=True\n```\n\nConfirm that telemetry is shut off through info logs streamed with engine invocation by looking for the phrase \"Skipping Neural Magic's latest package version check.\"\n\n## Community\n\n### Get In Touch\n\n- [Contribution Guide](https://github.com/neuralmagic/deepsparse/blob/main/CONTRIBUTING.md)\n- [Community Slack](https://neuralmagic.com/community/)\n- [GitHub Issue Queue](https://github.com/neuralmagic/deepsparse/issues) \n- [Subscribe To Our Newsletter](https://neuralmagic.com/subscribe/)\n- [Blog](https://www.neuralmagic.com/blog/) \n\nFor more general questions about Neural Magic, [complete this form.](http://neuralmagic.com/contact/)\n\n### License\n\n- **DeepSparse Community** is free to use and is licensed under the [Neural Magic DeepSparse Community License.](https://github.com/neuralmagic/deepsparse/blob/main/LICENSE)\nSome source code, example files, and scripts included in the DeepSparse GitHub repository or directory are licensed under the [Apache License Version 2.0](https://www.apache.org/licenses/LICENSE-2.0) as noted.\n\n- **DeepSparse Enterprise** requires a Trial License or [can be fully licensed](https://neuralmagic.com/legal/master-software-license-and-service-agreement/) for production, commercial applications.\n\n### Cite\n\nFind this project useful in your research or other communications? Please consider citing:\n\n```bibtex\n@misc{kurtic2023sparse,\n      title={Sparse Fine-Tuning for Inference Acceleration of Large Language Models}, \n      author={Eldar Kurtic and Denis Kuznedelev and Elias Frantar and Michael Goin and Dan Alistarh},\n      year={2023},\n      url={https://arxiv.org/abs/2310.06927},\n      eprint={2310.06927},\n      archivePrefix={arXiv},\n      primaryClass={cs.CL}\n}\n\n@misc{kurtic2022optimal,\n      title={The Optimal BERT Surgeon: Scalable and Accurate Second-Order Pruning for Large Language Models}, \n      author={Eldar Kurtic and Daniel Campos and Tuan Nguyen and Elias Frantar and Mark Kurtz and Benjamin Fineran and Michael Goin and Dan Alistarh},\n      year={2022},\n      url={https://arxiv.org/abs/2203.07259},\n      eprint={2203.07259},\n      archivePrefix={arXiv},\n      primaryClass={cs.CL}\n}\n\n@InProceedings{\n    pmlr-v119-kurtz20a, \n    title = {Inducing and Exploiting Activation Sparsity for Fast Inference on Deep Neural Networks}, \n    author = {Kurtz, Mark and Kopinsky, Justin and Gelashvili, Rati and Matveev, Alexander and Carr, John and Goin, Michael and Leiserson, William and Moore, Sage and Nell, Bill and Shavit, Nir and Alistarh, Dan}, \n    booktitle = {Proceedings of the 37th International Conference on Machine Learning}, \n    pages = {5533--5543}, \n    year = {2020}, \n    editor = {Hal Daumé III and Aarti Singh}, \n    volume = {119}, \n    series = {Proceedings of Machine Learning Research}, \n    address = {Virtual}, \n    month = {13--18 Jul}, \n    publisher = {PMLR}, \n    pdf = {http://proceedings.mlr.press/v119/kurtz20a/kurtz20a.pdf},\n    url = {http://proceedings.mlr.press/v119/kurtz20a.html}\n}\n\n@article{DBLP:journals/corr/abs-2111-13445,\n  author    = {Eugenia Iofinova and Alexandra Peste and Mark Kurtz and Dan Alistarh},\n  title     = {How Well Do Sparse Imagenet Models Transfer?},\n  journal   = {CoRR},\n  volume    = {abs/2111.13445},\n  year      = {2021},\n  url       = {https://arxiv.org/abs/2111.13445},\n  eprinttype = {arXiv},\n  eprint    = {2111.13445},\n  timestamp = {Wed, 01 Dec 2021 15:16:43 +0100},\n  biburl    = {https://dblp.org/rec/journals/corr/abs-2111-13445.bib},\n  bibsource = {dblp computer science bibliography, https://dblp.org}\n}\n```\n# All Thanks To Our Contributors\n\n\u003ca href=\"https://github.com/neuralmagic/deepsparse/graphs/contributors\"\u003e\n  \u003cimg src=\"https://contrib.rocks/image?repo=neuralmagic/deepsparse\" /\u003e\n\u003c/a\u003e\n","project_url":"https://awesome.ecosyste.ms/api/v1/projects/github.com%2Fneuralmagic%2Fdeepsparse","html_url":"https://awesome.ecosyste.ms/projects/github.com%2Fneuralmagic%2Fdeepsparse","lists_url":"https://awesome.ecosyste.ms/api/v1/projects/github.com%2Fneuralmagic%2Fdeepsparse/lists"}