{"id":19054659,"url":"https://github.com/ssbuild/bert_pretty","last_synced_at":"2025-04-24T03:23:08.361Z","repository":{"id":52085077,"uuid":"450367177","full_name":"ssbuild/bert_pretty","owner":"ssbuild","description":"bert text encode or decode","archived":false,"fork":false,"pushed_at":"2023-02-02T08:25:58.000Z","size":51,"stargazers_count":2,"open_issues_count":0,"forks_count":2,"subscribers_count":1,"default_branch":"master","last_synced_at":"2025-04-09T15:22:49.133Z","etag":null,"topics":["bert-text"],"latest_commit_sha":null,"homepage":"","language":"Python","has_issues":true,"has_wiki":null,"has_pages":null,"mirror_url":null,"source_name":null,"license":null,"status":null,"scm":"git","pull_requests_enabled":true,"icon_url":"https://github.com/ssbuild.png","metadata":{"files":{"readme":"README.md","changelog":null,"contributing":null,"funding":null,"license":null,"code_of_conduct":null,"threat_model":null,"audit":null,"citation":null,"codeowners":null,"security":null,"support":null}},"created_at":"2022-01-21T05:38:56.000Z","updated_at":"2024-02-29T04:57:17.000Z","dependencies_parsed_at":"2023-02-17T17:00:56.294Z","dependency_job_id":null,"html_url":"https://github.com/ssbuild/bert_pretty","commit_stats":null,"previous_names":["ssbuild/bert_text_pretty"],"tags_count":4,"template":false,"template_full_name":null,"repository_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/ssbuild%2Fbert_pretty","tags_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/ssbuild%2Fbert_pretty/tags","releases_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/ssbuild%2Fbert_pretty/releases","manifests_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/ssbuild%2Fbert_pretty/manifests","owner_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners/ssbuild","download_url":"https://codeload.github.com/ssbuild/bert_pretty/tar.gz/refs/heads/master","host":{"name":"GitHub","url":"https://github.com","kind":"github","repositories_count":249391025,"owners_count":21263205,"icon_url":"https://github.com/github.png","version":null,"created_at":"2022-05-30T11:31:42.601Z","updated_at":"2022-07-04T15:15:14.044Z","host_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub","repositories_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories","repository_names_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repository_names","owners_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners"}},"keywords":["bert-text"],"created_at":"2024-11-08T23:39:17.371Z","updated_at":"2025-04-17T23:32:29.536Z","avatar_url":"https://github.com/ssbuild.png","language":"Python","funding_links":[],"categories":[],"sub_categories":[],"readme":"### demo\r\n\r\n```python\r\n\r\nimport numpy as np\r\n#FullTokenizer is official and you can use your tokenization .\r\nfrom bert_pretty import FullTokenizer,\\\r\n        text_feature, \\\r\n        text_feature_char_level,\\\r\n        text_feature_word_level,\\\r\n        text_feature_char_level_input_ids_mask, \\\r\n        text_feature_word_level_input_ids_mask, \\\r\n        text_feature_char_level_input_ids_segment, \\\r\n        text_feature_word_level_input_ids_segment, \\\r\n        seqs_padding,rematch\r\n\r\n\r\nfrom bert_pretty.ner import load_label_bioes,load_label_bio,load_labels as ner_load_labels\r\nfrom bert_pretty.ner import ner_crf_decoding,\\\r\n                            ner_pointer_decoding,\\\r\n                            ner_pointer_decoding_with_mapping,\\\r\n                            ner_pointer_double_decoding,ner_pointer_double_decoding_with_mapping\r\n\r\nfrom bert_pretty.cls import cls_softmax_decoding,cls_sigmoid_decoding,load_labels as cls_load_labels\r\n\r\n\r\ntokenizer = FullTokenizer(vocab_file=r'F:\\pretrain\\chinese_L-12_H-768_A-12\\vocab.txt',do_lower_case=True)\r\ntext_list = [\"你是谁123aa\\ta嘂a\",\"嘂adasd\"]\r\n\r\n\r\ndef test():\r\n    do_lower_case = tokenizer.basic_tokenizer.do_lower_case\r\n    inputs = [tokenizer.tokenize(text) for text in text_list]\r\n\r\n    mapping = [rematch(text,tokens, do_lower_case) for text,tokens in zip(text_list,inputs)]\r\n\r\n    inputs = [ tokenizer.convert_tokens_to_ids(input) for input in inputs]\r\n\r\n    input_mask = [[1] * len(input) for input in inputs]\r\n    input_segment = [[0] * len(input) for input in inputs]\r\n    input_ids = seqs_padding(inputs)\r\n    input_mask = seqs_padding(input_mask)\r\n    input_segment = seqs_padding(input_segment)\r\n\r\n\r\n    print('input_ids\\n', input_ids)\r\n    print('mapping\\n',mapping)\r\n    print('input_mask\\n',input_mask)\r\n    print('input_segment\\n',input_segment)\r\n    print('\\n\\n')\r\n\r\n\r\n\r\ndef test_charlevel():\r\n    do_lower_case = tokenizer.basic_tokenizer.do_lower_case\r\n    vocab = tokenizer.vocab\r\n    unk_id = vocab.get('[UNK]')\r\n\r\n    inputs = []\r\n    for text in text_list:\r\n        if do_lower_case:\r\n            inputs.append([vocab.get(char,unk_id) for char in text])\r\n        else:\r\n            inputs.append([vocab.get(char.lower(), unk_id) for char in text])\r\n\r\n\r\n\r\n    inputs = [ tokenizer.convert_tokens_to_ids(input) for input in inputs]\r\n\r\n    input_mask = [[1] * len(input) for input in inputs]\r\n    input_segment = [[0] * len(input) for input in inputs]\r\n    input_ids = seqs_padding(inputs)\r\n    input_mask = seqs_padding(input_mask)\r\n    input_segment = seqs_padding(input_segment)\r\n\r\n    print('input_ids\\n', input_ids)\r\n    print('input_mask\\n',input_mask)\r\n    print('input_segment\\n',input_segment)\r\n    print('\\n\\n')\r\n\r\n# labels = ['标签1','标签2']\r\n# print(cls.load_labels(labels))\r\n#\r\n# print(ner.load_label_bio(labels))\r\n\r\n\r\n'''\r\n    # def ner_crf_decoding(batch_text, id2label, batch_logits, trans=None,batch_mapping=None,with_dict=True):\r\n    ner crf decode 解析crf序列  or 解析 已经解析过的crf序列\r\n\r\n    batch_text input_instance list ,\r\n    id2label 标签 list or dict\r\n    batch_logits 为bert 预测结果 logits_all (batch,seq_len,num_tags) or (batch,seq_len)\r\n    trans 是否启用trans预测 , 2D\r\n    batch_mapping 映射序列\r\n'''\r\n\r\n'''\r\n    def ner_pointer_decoding(batch_text, id2label, batch_logits, threshold=1e-8,coordinates_minus=False,with_dict=True)\r\n\r\n    batch_text text list ,\r\n    id2label 标签 list or dict\r\n    batch_logits (batch,num_labels,seq_len,seq_len)\r\n    threshold 阈值\r\n    coordinates_minus\r\n'''\r\n\r\n'''\r\n    def ner_pointer_decoding_with_mapping(batch_text, id2label, batch_logits, batch_mapping,threshold=1e-8,coordinates_minus=False,with_dict=True)\r\n\r\n    batch_text text list ,\r\n    id2label 标签 list or dict\r\n    batch_logits (batch,num_labels,seq_len,seq_len)\r\n    threshold 阈值\r\n    coordinates_minus\r\n'''\r\n\r\n\r\n'''\r\n    cls_softmax_decoding(batch_text, id2label, batch_logits,threshold=None)\r\n    batch_text 文本list ,\r\n    id2label 标签 list or dict\r\n    batch_logits (batch,num_classes)\r\n    threshold 阈值\r\n'''\r\n\r\n'''\r\n    cls_sigmoid_decoding(batch_text, id2label, batch_logits,threshold=0.5)\r\n\r\n    batch_text 文本list ,\r\n    id2label 标签 list or dict\r\n    batch_logits (batch,num_classes)\r\n    threshold 阈值\r\n'''\r\n\r\n\r\ndef test_cls_decode():\r\n    num_label =3\r\n    np.random.seed(123)\r\n    batch_logits = np.random.rand(2,num_label)\r\n    result = cls_softmax_decoding(text_list,['标签1','标签2','标签3'],batch_logits,threshold=None)\r\n    print(result)\r\n\r\n\r\n    batch_logits = np.random.rand(2,num_label)\r\n    print(batch_logits)\r\n    result = cls_sigmoid_decoding(text_list,['标签1','标签2','标签3'],batch_logits,threshold=0.5)\r\n    print(result)\r\n\r\n\r\n\r\nif __name__ == '__main__':\r\n    test()\r\n    test_charlevel()\r\n\r\n    test_cls_decode()\r\n \r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n```\r\n","project_url":"https://awesome.ecosyste.ms/api/v1/projects/github.com%2Fssbuild%2Fbert_pretty","html_url":"https://awesome.ecosyste.ms/projects/github.com%2Fssbuild%2Fbert_pretty","lists_url":"https://awesome.ecosyste.ms/api/v1/projects/github.com%2Fssbuild%2Fbert_pretty/lists"}