{"id":50729112,"url":"https://github.com/dettinjo/llm-fact-auditor","last_synced_at":"2026-06-10T07:01:51.223Z","repository":{"id":362261323,"uuid":"1076401023","full_name":"dettinjo/LLM-Fact-Auditor","owner":"dettinjo","description":"A post-processing pipeline to fact-check, entity-link, and verify answers from Large Language Models (LLMs). Developed for the Web Data Processing Systems course at Vrije Universiteit Amsterdam.","archived":false,"fork":false,"pushed_at":"2026-06-03T09:48:17.000Z","size":948,"stargazers_count":0,"open_issues_count":0,"forks_count":0,"subscribers_count":0,"default_branch":"main","last_synced_at":"2026-06-03T11:22:29.380Z","etag":null,"topics":["data-processing","docker","entity-linking","fact-checking","llm","nlp","portfolio","python","pytorch","question-answering","spacy","stanza","transformers","wikidata"],"latest_commit_sha":null,"homepage":"","language":"Python","has_issues":true,"has_wiki":null,"has_pages":null,"mirror_url":null,"source_name":null,"license":"mit","status":null,"scm":"git","pull_requests_enabled":true,"icon_url":"https://github.com/dettinjo.png","metadata":{"files":{"readme":"README.md","changelog":null,"contributing":null,"funding":null,"license":"LICENSE","code_of_conduct":null,"threat_model":null,"audit":null,"citation":null,"codeowners":null,"security":null,"support":null,"governance":null,"roadmap":null,"authors":null,"dei":null,"publiccode":null,"codemeta":null,"zenodo":null,"notice":null,"maintainers":null,"copyright":null,"agents":null,"dco":null,"cla":null}},"created_at":"2025-10-14T20:04:49.000Z","updated_at":"2026-06-03T09:48:21.000Z","dependencies_parsed_at":null,"dependency_job_id":null,"html_url":"https://github.com/dettinjo/LLM-Fact-Auditor","commit_stats":null,"previous_names":["dettinjo/llm-fact-auditor"],"tags_count":null,"template":false,"template_full_name":null,"purl":"pkg:github/dettinjo/LLM-Fact-Auditor","repository_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/dettinjo%2FLLM-Fact-Auditor","tags_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/dettinjo%2FLLM-Fact-Auditor/tags","releases_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/dettinjo%2FLLM-Fact-Auditor/releases","manifests_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/dettinjo%2FLLM-Fact-Auditor/manifests","owner_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners/dettinjo","download_url":"https://codeload.github.com/dettinjo/LLM-Fact-Auditor/tar.gz/refs/heads/main","sbom_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/dettinjo%2FLLM-Fact-Auditor/sbom","scorecard":null,"host":{"name":"GitHub","url":"https://github.com","kind":"github","repositories_count":286080680,"owners_count":34140776,"icon_url":"https://github.com/github.png","version":null,"created_at":"2022-05-30T11:31:42.601Z","updated_at":"2026-05-26T15:22:16.424Z","status":"online","status_checked_at":"2026-06-10T02:00:07.152Z","response_time":89,"last_error":null,"robots_txt_status":"success","robots_txt_updated_at":"2025-07-24T06:49:26.215Z","robots_txt_url":"https://github.com/robots.txt","online":true,"can_crawl_api":true,"host_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub","repositories_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories","repository_names_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repository_names","owners_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners"}},"keywords":["data-processing","docker","entity-linking","fact-checking","llm","nlp","portfolio","python","pytorch","question-answering","spacy","stanza","transformers","wikidata"],"created_at":"2026-06-10T07:01:48.804Z","updated_at":"2026-06-10T07:01:51.218Z","avatar_url":"https://github.com/dettinjo.png","language":"Python","funding_links":[],"categories":[],"sub_categories":[],"readme":"\u003c!-- Improved compatibility of back to top link --\u003e\n\u003ca id=\"readme-top\"\u003e\u003c/a\u003e\n\n\u003c!-- PROJECT SHIELDS --\u003e\n\u003cdiv align=\"center\"\u003e\n  \u003ca href=\"https://github.com/dettinjo/LLM-Fact-Auditor/graphs/contributors\"\u003e\n    \u003cimg alt=\"Contributors\" src=\"https://img.shields.io/github/contributors/dettinjo/LLM-Fact-Auditor.svg?style=for-the-badge\"\u003e\n  \u003c/a\u003e\n  \u003ca href=\"https://github.com/dettinjo/LLM-Fact-Auditor/network/members\"\u003e\n    \u003cimg alt=\"Forks\" src=\"https://img.shields.io/github/forks/dettinjo/LLM-Fact-Auditor.svg?style=for-the-badge\"\u003e\n  \u003c/a\u003e\n  \u003ca href=\"https://github.com/dettinjo/LLM-Fact-Auditor/stargazers\"\u003e\n    \u003cimg alt=\"Stargazers\" src=\"https://img.shields.io/github/stars/dettinjo/LLM-Fact-Auditor.svg?style=for-the-badge\"\u003e\n  \u003c/a\u003e\n  \u003ca href=\"https://github.com/dettinjo/LLM-Fact-Auditor/issues\"\u003e\n    \u003cimg alt=\"Issues\" src=\"https://img.shields.io/github/issues/dettinjo/LLM-Fact-Auditor.svg?style=for-the-badge\"\u003e\n  \u003c/a\u003e\n  \u003ca href=\"https://github.com/dettinjo/LLM-Fact-Auditor/blob/main/LICENSE\"\u003e\n    \u003cimg alt=\"MIT License\" src=\"https://img.shields.io/github/license/dettinjo/LLM-Fact-Auditor.svg?style=for-the-badge\"\u003e\n  \u003c/a\u003e\n\u003c/div\u003e\n\n\u003c!-- PROJECT LOGO --\u003e\n\u003cbr /\u003e\n\u003cdiv align=\"center\"\u003e\n  \u003ch3 align=\"center\"\u003eLLM Fact Auditor\u003c/h3\u003e\n\n  \u003cp align=\"center\"\u003e\n    A post-processing pipeline to fact-check, entity-link, and verify answers from Large Language Models.\n    \u003cbr /\u003e\n    \u003cbr /\u003e\n    \u003ca href=\"#about-the-project\"\u003eAbout the Project\u003c/a\u003e\n    \u0026middot;\n    \u003ca href=\"#getting-started\"\u003eGetting Started\u003c/a\u003e\n    \u0026middot;\n    \u003ca href=\"#usage\"\u003eUsage\u003c/a\u003e\n  \u003c/p\u003e\n\u003c/div\u003e\n\n\u003c!-- TABLE OF CONTENTS --\u003e\n\u003cdetails\u003e\n  \u003csummary\u003eTable of Contents\u003c/summary\u003e\n  \u003col\u003e\n    \u003cli\u003e\n      \u003ca href=\"#about-the-project\"\u003eAbout The Project\u003c/a\u003e\n      \u003cul\u003e\n        \u003cli\u003e\u003ca href=\"#built-with\"\u003eBuilt With\u003c/a\u003e\u003c/li\u003e\n      \u003c/ul\u003e\n    \u003c/li\u003e\n    \u003cli\u003e\n      \u003ca href=\"#getting-started\"\u003eGetting Started\u003c/a\u003e\n      \u003cul\u003e\n        \u003cli\u003e\u003ca href=\"#prerequisites\"\u003ePrerequisites\u003c/a\u003e\u003c/li\u003e\n        \u003cli\u003e\u003ca href=\"#installation\"\u003eInstallation\u003c/a\u003e\u003c/li\u003e\n      \u003c/ul\u003e\n    \u003c/li\u003e\n    \u003cli\u003e\u003ca href=\"#usage\"\u003eUsage\u003c/a\u003e\u003c/li\u003e\n    \u003cli\u003e\u003ca href=\"#roadmap\"\u003eRoadmap\u003c/a\u003e\u003c/li\u003e\n    \u003cli\u003e\u003ca href=\"#license\"\u003eLicense\u003c/a\u003e\u003c/li\u003e\n    \u003cli\u003e\u003ca href=\"#contact\"\u003eContact\u003c/a\u003e\u003c/li\u003e\n    \u003cli\u003e\u003ca href=\"#acknowledgments\"\u003eAcknowledgments\u003c/a\u003e\u003c/li\u003e\n  \u003c/ol\u003e\n\u003c/details\u003e\n\n\u003c!-- ABOUT THE PROJECT --\u003e\n## About The Project\n\nLarge Language Models (LLMs) are powerful, but they can produce factually incorrect or unverifiable information—a phenomenon often called \"hallucination.\" This project, **LLM Fact Auditor**, serves as a robust post-processing pipeline designed to address this challenge. It takes a question and a raw LLM-generated answer, then enriches and verifies it through a multi-stage process.\n\nHere's what it does:\n*   **Entity Linking**: It identifies named entities (like people, places, and organizations) in the text and links them to their corresponding Wikipedia pages, grounding the response in factual data.\n*   **Answer Extraction**: It distills the often verbose LLM response into a concise, direct answer, such as a \"yes/no\" or a specific entity.\n*   **Fact-Checking**: It verifies the extracted answer's correctness by cross-referencing it with structured knowledge from Wikidata and the content of the linked Wikipedia pages.\n\nThis system was developed as a university project to create a practical tool for improving the reliability of AI-generated content.\n\n\u003cp align=\"right\"\u003e(\u003ca href=\"#readme-top\"\u003eback to top\u003c/a\u003e)\u003c/p\u003e\n\n### Built With\n\nThis project leverages a powerful stack of modern NLP tools and libraries.\n\n\u003cp\u003e\n  \u003ca href=\"https://www.python.org/\"\u003e\n    \u003cimg alt=\"Python\" src=\"https://img.shields.io/badge/python-3776AB?style=for-the-badge\u0026logo=python\u0026logoColor=white\"\u003e\n  \u003c/a\u003e\n  \u003ca href=\"https://www.docker.com/\"\u003e\n    \u003cimg alt=\"Docker\" src=\"https://img.shields.io/badge/Docker-2496ED?style=for-the-badge\u0026logo=docker\u0026logoColor=white\"\u003e\n  \u003c/a\u003e\n  \u003ca href=\"https://llama.meta.com/\"\u003e\n    \u003cimg alt=\"Llama\" src=\"https://img.shields.io/badge/LLama-2396F3?style=for-the-badge\u0026logo=meta\u0026logoColor=white\"\u003e\n  \u003c/a\u003e\n  \u003ca href=\"https://pytorch.org/\"\u003e\n    \u003cimg alt=\"PyTorch\" src=\"https://img.shields.io/badge/PyTorch-EE4C2C?style=for-the-badge\u0026logo=pytorch\u0026logoColor=white\"\u003e\n  \u003c/a\u003e\n  \u003ca href=\"https://huggingface.co/docs/transformers/index\"\u003e\n    \u003cimg alt=\"Transformers\" src=\"https://img.shields.io/badge/Transformers-FFD21E?style=for-the-badge\u0026logo=huggingface\u0026logoColor=black\"\u003e\n  \u003c/a\u003e\n  \u003ca href=\"https://spacy.io/\"\u003e\n    \u003cimg alt=\"spaCy\" src=\"https://img.shields.io/badge/spaCy-09A3D5?style=for-the-badge\u0026logo=spacy\u0026logoColor=white\"\u003e\n  \u003c/a\u003e\n\u003c/p\u003e\n\n\u003cp align=\"right\"\u003e(\u003ca href=\"#readme-top\"\u003eback to top\u003c/a\u003e)\u003c/p\u003e\n\n\u003c!-- GETTING STARTED --\u003e\n## Getting Started\n\nFollow these steps to set up and run the project locally within the provided Docker environment.\n\n### Prerequisites\n\n*   **Docker**: You must have Docker installed and running.\n*   **WDPS Docker Image**: The project is designed to run inside the `karmaresearch/wdps2` Docker container. Ensure you have this container running.\n    ```sh\n    docker ps\n    ```\n\n### Installation\n\n1.  **Clone the Repository**:\n    ```sh\n    git clone https://github.com/dettinjo/LLM-Fact-Auditor.git\n    cd LLM-Fact-Auditor\n    ```\n2.  **Copy Project Files to Docker**: From your host machine's terminal, copy the entire project directory into your running Docker container.\n    ```sh\n    docker cp ./ \u003ccontainer_id\u003e:/home/user/submission\n    ```\n3.  **Access the Container and Set Up Environment**:\n    ```sh\n    # Enter the container's shell\n    docker exec -it \u003ccontainer_id\u003e bash\n\n    # Navigate to the project directory\n    cd /home/user/submission\n\n    # Switch to root user to install dependencies\n    sudo su\n\n    # Create and activate a virtual environment\n    python3 -m venv virtual_env\n    source virtual_env/bin/activate\n    ```\n4.  **Install Dependencies**: Install all required Python packages and download the necessary NLP models. This step may take some time.\n    ```sh\n    # Install Python packages\n    pip install -r requirements.txt\n\n    # Run the setup script to download all models\n    python src/setup.py\n    ```\n\n\u003cp align=\"right\"\u003e(\u003ca href=\"#readme-top\"\u003eback to top\u003c/a\u003e)\u003c/p\u003e\n\n\u003c!-- USAGE EXAMPLES --\u003e\n## Usage\n\nThe main script is designed to read questions from standard input and write the processed output to standard output.\n\n### Running with Llama 2 (Default)\nThis command reads questions from `test_data/input.txt` and saves the structured output to `test_data/output.txt`.\n\n```sh\npython3 main.py \u003c ./test_data/input.txt \u003e ./test_data/output.txt\n```\n\n### Running with Llama 3\nFor higher quality answers and faster performance, you can use the Llama 3 model by adding the `--llama_ver=3` flag.\n\n```sh\npython3 main.py --llama_ver=3 \u003c ./test_data/input.txt \u003e ./test_data/output.txt\n```\n\n### Example Input \u0026 Output\n\n**Input Question in `input.txt`:**\n```\nquestion-001\tIs Managua the capital of Nicaragua?\n```\n\n**Corresponding Output in `output.txt`:**\n```\nquestion-001\tR\"Yes, Managua is the capital and largest city of Nicaragua.\"\nquestion-001\tA\"yes\"\nquestion-001\tC\"correct\"\nquestion-001\tE\"Managua\"\t\"https://en.wikipedia.org/wiki/Managua\"\nquestion-001\tE\"Nicaragua\"\t\"https://en.wikipedia.org/wiki/Nicaragua\"\n```\n\nThe output format includes the raw **R**esponse, extracted **A**nswer, **C**orrectness check, and linked **E**ntities.\n\n\u003cp align=\"right\"\u003e(\u003ca href=\"#readme-top\"\u003eback to top\u003c/a\u003e)\u003c/p\u003e\n\n\u003c!-- ROADMAP --\u003e\n## Roadmap\n\n- [ ] Implement a more robust relation extraction module.\n- [ ] Add support for additional knowledge bases beyond Wikidata.\n- [ ] Develop a simple web interface for interactive demonstrations.\n- [ ] Expand fact-checking capabilities to handle more complex and nuanced claims.\n\nSee the [open issues](https://github.com/dettinjo/LLM-Fact-Auditor/issues) for a full list of proposed features (and known issues).\n\n\u003cp align=\"right\"\u003e(\u003ca href=\"#readme-top\"\u003eback to top\u003c/a\u003e)\u003c/p\u003e\n\n\u003c!-- LICENSE --\u003e\n## License\n\nDistributed under the MIT License. See `LICENSE` for more information.\n\n\u003cp align=\"right\"\u003e(\u003ca href=\"#readme-top\"\u003eback to top\u003c/a\u003e)\u003c/p\u003e\n\n\u003c!-- CONTACT --\u003e\n## Contact\n\nThis project was created for the Web Data Processing Systems course (XM_40020) at Vrije Universiteit Amsterdam.\n\n**Group Members:**\n*   Joel Dettinger - j.dettinger@student.vu.nl\n*   Ruida Zhou - r.zhou4@student.vu.nl\n*   Hongqian Xia - h.xia@student.vu.nl\n*   Angelo De Nadai - a.denadai@student.vu.nl\n\nProject Link: [https://github.com/dettinjo/LLM-Fact-Auditor](https://github.com/dettinjo/LLM-Fact-Auditor)\n\n\u003cp align=\"right\"\u003e(\u003ca href=\"#readme-top\"\u003eback to top\u003c/a\u003e)\u003c/p\u003e\n\n\u003c!-- ACKNOWLEDGMENTS --\u003e\n## Acknowledgments\n\n*   Vrije Universiteit Amsterdam\n*   Hugging Face for the incredible `transformers` library and model hosting.\n*   The developers of spaCy, Stanza, and the Wikidata platform.\n*   [Othneil Drew's Best-README-Template](https://github.com/othneildrew/Best-README-Template)\n\n\u003cp align=\"right\"\u003e(\u003ca href=\"#readme-top\"\u003eback to top\u003c/a\u003e)\u003c/p\u003e\n","project_url":"https://awesome.ecosyste.ms/api/v1/projects/github.com%2Fdettinjo%2Fllm-fact-auditor","html_url":"https://awesome.ecosyste.ms/projects/github.com%2Fdettinjo%2Fllm-fact-auditor","lists_url":"https://awesome.ecosyste.ms/api/v1/projects/github.com%2Fdettinjo%2Fllm-fact-auditor/lists"}