{"id":27232963,"url":"https://github.com/mlcommons/tokenizer","last_synced_at":"2025-04-10T14:11:13.673Z","repository":null,"keywords":[],"created_at":"2025-04-10T14:11:10.698Z","updated_at":"2025-04-10T14:11:10.698Z","avatar_url":null,"language":null,"funding_links":[],"categories":["🔹 **WordPiece Tokenizer Implementations**"],"sub_categories":[],"readme":null,"project_url":"https://awesome.ecosyste.ms/api/v1/projects/github.com%2Fmlcommons%2Ftokenizer","html_url":"https://awesome.ecosyste.ms/projects/github.com%2Fmlcommons%2Ftokenizer","lists_url":"https://awesome.ecosyste.ms/api/v1/projects/github.com%2Fmlcommons%2Ftokenizer/lists"}