{"id":22108861,"url":"https://github.com/hoanglayor/finetune-llms-for-sequence-classification","last_synced_at":"2026-01-26T02:01:58.499Z","repository":{"id":252682032,"uuid":"841126371","full_name":"HoangLayor/Finetune-LLMs-for-Sequence-Classification","owner":"HoangLayor","description":"Finetune Flan-T5 for email classification","archived":false,"fork":false,"pushed_at":"2024-09-12T08:39:48.000Z","size":138,"stargazers_count":3,"open_issues_count":0,"forks_count":0,"subscribers_count":1,"default_branch":"main","last_synced_at":"2025-10-12T04:32:06.206Z","etag":null,"topics":["email-classification","flan-t5","spam-classification"],"latest_commit_sha":null,"homepage":"","language":"Jupyter Notebook","has_issues":true,"has_wiki":null,"has_pages":null,"mirror_url":null,"source_name":null,"license":null,"status":null,"scm":"git","pull_requests_enabled":true,"icon_url":"https://github.com/HoangLayor.png","metadata":{"files":{"readme":"README.md","changelog":null,"contributing":null,"funding":null,"license":null,"code_of_conduct":null,"threat_model":null,"audit":null,"citation":null,"codeowners":null,"security":null,"support":null,"governance":null,"roadmap":null,"authors":null,"dei":null,"publiccode":null,"codemeta":null}},"created_at":"2024-08-11T18:03:27.000Z","updated_at":"2024-10-24T10:41:00.000Z","dependencies_parsed_at":"2024-08-11T19:28:48.943Z","dependency_job_id":"954c90f4-ebce-4832-b78a-55d30720d183","html_url":"https://github.com/HoangLayor/Finetune-LLMs-for-Sequence-Classification","commit_stats":null,"previous_names":["hoanglayor/finetune-llms-for-sequence-classification"],"tags_count":0,"template":false,"template_full_name":null,"purl":"pkg:github/HoangLayor/Finetune-LLMs-for-Sequence-Classification","repository_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/HoangLayor%2FFinetune-LLMs-for-Sequence-Classification","tags_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/HoangLayor%2FFinetune-LLMs-for-Sequence-Classification/tags","releases_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/HoangLayor%2FFinetune-LLMs-for-Sequence-Classification/releases","manifests_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/HoangLayor%2FFinetune-LLMs-for-Sequence-Classification/manifests","owner_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners/HoangLayor","download_url":"https://codeload.github.com/HoangLayor/Finetune-LLMs-for-Sequence-Classification/tar.gz/refs/heads/main","sbom_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/HoangLayor%2FFinetune-LLMs-for-Sequence-Classification/sbom","scorecard":null,"host":{"name":"GitHub","url":"https://github.com","kind":"github","repositories_count":286080680,"owners_count":28764408,"icon_url":"https://github.com/github.png","version":null,"created_at":"2022-05-30T11:31:42.601Z","updated_at":"2026-01-26T00:37:26.264Z","status":"online","status_checked_at":"2026-01-26T02:00:08.215Z","response_time":59,"last_error":null,"robots_txt_status":"success","robots_txt_updated_at":"2025-07-24T06:49:26.215Z","robots_txt_url":"https://github.com/robots.txt","online":true,"can_crawl_api":true,"host_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub","repositories_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories","repository_names_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repository_names","owners_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners"}},"keywords":["email-classification","flan-t5","spam-classification"],"created_at":"2024-12-01T09:20:39.602Z","updated_at":"2026-01-26T02:01:58.481Z","avatar_url":"https://github.com/HoangLayor.png","language":"Jupyter Notebook","funding_links":[],"categories":[],"sub_categories":[],"readme":"***Finetune Flan-T5 for email classification***\n# Prepare data\n- Xử lý email và đưa vào file csv\n```python\nimport pandas as pd\nimport email\nimport csv\nfrom email import policy\nimport os\nimport re\n\n# Sắp xếp file theo thứ tự\ndef natural_sort_key(s):\n    return [int(c) if c.isdigit() else c.lower() for c in re.split(r'(\\d+)', s)]\n\ndef pandas_escape(text):\n    df = pd.DataFrame([text])\n    return df.to_csv(index=False, header=False, quoting=csv.QUOTE_ALL, escapechar='\\\\').strip()\n\ndef email_to_string(email_content):\n    msg = email.message_from_string(email_content, policy=policy.default)\n\n    headers = ['From', 'To', 'Subject', 'Date']\n    email_data = \"\"\n    for header in headers:\n        if msg.get(header, '') == '':\n            continue\n        email_data += f\"{header}: {msg.get(header, '')}\\n\"\n\n    body = ''\n    if msg.is_multipart():\n        for part in msg.walk():\n            if part.get_content_type() == \"text/plain\":\n                body = part.get_payload(decode=True).decode()\n                break\n    else:\n        body = msg.get_payload(decode=True).decode()\n\n    body = body.rstrip('\\n')\n    full_email = f\"{email_data}Body: {body}\"\n    return pandas_escape(full_email)\n\ndef custom_data_to_csv(label, folder_path, csv_file):\n    for file_name in sorted(os.listdir(folder_path), key=natural_sort_key):\n        file_path = os.path.join(folder_path, file_name)\n        email_str = ''\n        with open(file_path, 'r') as f:\n            email_str = f.read()\n            email_str = email_to_string(email_str)\n        with open(csv_file, 'a') as f:\n            f.write(f\"{file_name},{label},{email_str}\\n\")\n```\n```python\nROOT_DIR = \"/teamspace/studios/this_studio\"\n\nspam_data_dir = os.path.join(ROOT_DIR, \"data/TrainData/spam\")\nnotspam_data_dir = os.path.join(ROOT_DIR, \"data/TrainData/notspam\")\ntest_data_dir = os.path.join(ROOT_DIR, \"data/TestData_nolabel\")\n\ntrain_csv_file = os.path.join(ROOT_DIR, \"BaiThi2/train_data.csv\")\ntest_csv_file = os.path.join(ROOT_DIR, \"BaiThi2/test_data.csv\")\n```\n```python\n# from custom_data import custom_data_to_csv\n\nwith open(train_csv_file, 'w') as csv_f:\n    csv_f.write(\"filename,label,text\\n\")\n\ncustom_data_to_csv(\"spam\", spam_data_dir, train_csv_file)\ncustom_data_to_csv(\"notspam\", notspam_data_dir, train_csv_file)\ndf = pd.read_csv(train_csv_file)\nprint(df.head())\nprint(df.info())\ndf = df.drop(columns=['filename'])\ndf\n```\n# Load dataset\n```python\nimport os\nimport numpy as np\nimport pandas as pd\nfrom datasets import Dataset\n\nlabel2id = {\"notspam\": 0, \"spam\": 1}\nid2label = {id: label for label, id in label2id.items()}\n\ndataset_email = pd.read_csv(train_csv_file)\ndataset_email = dataset_email.drop(columns=[\"filename\"])\n\ndef load_dataset(model_type: str = \"\") -\u003e Dataset:\n    \"\"\"Load dataset.\"\"\"\n\n    dataset_email[\"label\"] = dataset_email[\"label\"].astype(str)\n    if model_type == \"AutoModelForSequenceClassification\":\n        # Convert labels to integers\n        dataset_email[\"label\"] = dataset_email[\"label\"].map(\n            label2id\n        )\n\n    dataset_email[\"text\"] = dataset_email[\"text\"].astype(str)\n    dataset = Dataset.from_pandas(dataset_email)\n    dataset = dataset.shuffle(seed=42)\n    dataset = dataset.train_test_split(test_size=0.2)\n\n    return dataset\n```\n```python\n# from data_loader import load_dataset\n\ntrain_dataset = load_dataset(\"AutoModelForSequenceClassification\")\ntrain_dataset\n```\n```python\n# Initialize base model and tokenizer\nfrom transformers import (\n    AutoConfig,\n    AutoModelForSequenceClassification,\n    AutoTokenizer,\n    Trainer,\n    TrainingArguments,\n)\n\nlabel2id = {\"notspam\": 0, \"spam\": 1}\nid2label = {id: label for label, id in label2id.items()}\n\nMODEL_ID = \"google/flan-t5-base\"\nREPOSITORY_ID = f\"{MODEL_ID.split('/')[1]}-email-classification\"\n\nmodel = AutoModelForSequenceClassification.from_pretrained(MODEL_ID, num_labels=len(label2id))\ntokenizer = AutoTokenizer.from_pretrained(MODEL_ID)\n```\n```python\nimport evaluate\nimport nltk\nimport numpy as np\nfrom sklearn.metrics import precision_recall_fscore_support\n\nmetric = evaluate.load(\"accuracy\")\n\ndef tokenize_function(examples):\n   return tokenizer(examples[\"text\"], padding=\"max_length\", truncation=True)\n\ndef compute_metrics(eval_pred) -\u003e dict:\n   \"\"\"Compute metrics for evaluation\"\"\"\n   logits, labels = eval_pred\n   if isinstance(\n      logits, tuple\n   ):  # if the model also returns hidden_states or attentions\n      logits = logits[0]\n   predictions = np.argmax(logits, axis=-1)\n   precision, recall, f1, _ = precision_recall_fscore_support(\n      labels, predictions, average=\"binary\"\n   )\n   return {\"precision\": precision, \"recall\": recall, \"f1\": f1}\n```\n```python\ntokenized_datasets = train_dataset.map(tokenize_function, batched=True)\nprint(tokenized_datasets)\n\nsmall_train_dataset = tokenized_datasets[\"train\"].shuffle(seed=42)\nsmall_eval_dataset = tokenized_datasets[\"test\"].shuffle(seed=42)\n```\n# Training\n```python\ntraining_args = TrainingArguments(\n    num_train_epochs=5,\n    output_dir=REPOSITORY_ID,\n    logging_strategy=\"steps\",\n    logging_steps=100,\n    report_to=\"tensorboard\",\n    per_device_train_batch_size=8,\n    per_device_eval_batch_size=8,\n    fp16=False,  # Overflows with fp16\n    learning_rate=3e-4,\n    save_strategy=\"epoch\",\n    save_total_limit=2,\n    load_best_model_at_end=False,\n)\n\ntrainer = Trainer(\n   model=model,\n   args=training_args,\n   train_dataset=small_train_dataset,\n   eval_dataset=small_eval_dataset,\n   compute_metrics=compute_metrics,\n)\n```\n- Save model\n```python\ntrainer.train()\n```\n```python\ntokenizer.save_pretrained(REPOSITORY_ID)\nprint(trainer.evaluate())\n```\n# Evaluate\n- Load model\n```python\nimport torch\nfrom transformers import (\n    AutoModelForSequenceClassification,\n    AutoTokenizer,\n)\nmodel = AutoModelForSequenceClassification.from_pretrained(REPOSITORY_ID)\nmodel.to(\"cuda\") if torch.cuda.is_available() else model.to(\"cpu\")\n```\n- Classify \u0026 Eval\n```python\nfrom time import time\nfrom typing import List, Tuple\n\nimport torch\nfrom loguru import logger\nfrom sklearn.metrics import classification_report\nfrom tqdm.auto import tqdm\nfrom transformers import AutoModelForSequenceClassification, AutoTokenizer\n\ndef classify(texts_to_classify: List[str]) -\u003e List[Tuple[str, float]]:\n    \"\"\"Classify a list of texts using the model.\"\"\"\n    # Tokenize all texts in the batch\n    start = time()\n    inputs = tokenizer(\n        texts_to_classify,\n        return_tensors=\"pt\",\n        max_length=512,\n        truncation=True,\n        padding=True,\n    )\n    inputs = inputs.to(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n    # Get predictions\n    with torch.no_grad():\n        outputs = model(**inputs)\n    \n    logger.debug(\n        f\"Classification of {len(texts_to_classify)} examples took {time() - start} seconds\"\n    ) # logger\n\n    # Process the outputs to get the probability distribution\n    logits = outputs.logits\n    probs = torch.nn.functional.softmax(logits, dim=-1)\n\n    # Get the top class and the corresponding probability (certainty) for each input text\n    confidences, predicted_classes = torch.max(probs, dim=1)\n    predicted_classes = (\n        predicted_classes.cpu().numpy()\n    )  # Move to CPU for numpy conversion if needed\n    confidences = confidences.cpu().numpy()  # Same here\n\n    # Map predicted class IDs to labels\n    predicted_labels = [id2label[class_id] for class_id in predicted_classes]\n\n    # Zip together the predicted labels and confidences and convert to a list of tuples\n    return list(zip(predicted_labels, confidences))\n\ndef eval():\n    \"\"\"Evaluate the model on the test dataset.\"\"\"\n    predictions_list, labels_list = [], []\n\n    batch_size = 16  # Adjust batch size based GPU capacity\n    num_batches = len(train_dataset[\"test\"]) // batch_size + (\n        0 if len(train_dataset[\"test\"]) % batch_size == 0 else 1\n    )\n    progress_bar = tqdm(total=num_batches, desc=\"Evaluating\")\n\n    for i in range(0, len(train_dataset[\"test\"]), batch_size):\n        batch_texts = train_dataset[\"test\"][\"text\"][i : i + batch_size]\n        batch_labels = train_dataset[\"test\"][\"label\"][i : i + batch_size]\n\n        batch_predictions = classify(batch_texts)\n\n        predictions_list.extend(batch_predictions)\n        labels_list.extend([id2label[label_id] for label_id in batch_labels])\n\n        progress_bar.update(1)\n\n    progress_bar.close()\n    report = classification_report(labels_list, [pair[0] for pair in predictions_list])\n    print(report)\n\neval()\n```\n```python\npredictions_list, labels_list = [], []\n\ntexts = train_dataset['test']['text']\nlabels = train_dataset['test']['label']\n\npredictions = classify(texts)\n\npredictions_list.extend(predictions)\nlabels_list.extend([id2label[label_id] for label_id in labels])\nfor id, prediction in enumerate(predictions):\n    print(f\"Actual Label: {labels_list[id]}\\n\u003e\u003e\u003e Prediction: {predictions_list[id]}\")\n```\n","project_url":"https://awesome.ecosyste.ms/api/v1/projects/github.com%2Fhoanglayor%2Ffinetune-llms-for-sequence-classification","html_url":"https://awesome.ecosyste.ms/projects/github.com%2Fhoanglayor%2Ffinetune-llms-for-sequence-classification","lists_url":"https://awesome.ecosyste.ms/api/v1/projects/github.com%2Fhoanglayor%2Ffinetune-llms-for-sequence-classification/lists"}