{"id":20789186,"url":"https://github.com/umstek/dupkiller","last_synced_at":"2026-07-03T04:34:11.874Z","repository":{"id":53782705,"uuid":"77988405","full_name":"umstek/DupKiller","owner":"umstek","description":"Slow, but more reliable duplicate files cleaner. ","archived":false,"fork":false,"pushed_at":"2021-03-14T10:33:46.000Z","size":381,"stargazers_count":1,"open_issues_count":0,"forks_count":1,"subscribers_count":1,"default_branch":"master","last_synced_at":"2025-10-30T01:45:00.413Z","etag":null,"topics":["cleaner","duplicate-files","storage"],"latest_commit_sha":null,"homepage":null,"language":"C#","has_issues":true,"has_wiki":null,"has_pages":null,"mirror_url":null,"source_name":null,"license":"mit","status":null,"scm":"git","pull_requests_enabled":true,"icon_url":"https://github.com/umstek.png","metadata":{"files":{"readme":"README.md","changelog":null,"contributing":null,"funding":null,"license":"LICENSE","code_of_conduct":null,"threat_model":null,"audit":null,"citation":null,"codeowners":null,"security":null,"support":null}},"created_at":"2017-01-04T06:35:13.000Z","updated_at":"2025-10-04T13:56:57.000Z","dependencies_parsed_at":"2022-08-30T08:22:04.923Z","dependency_job_id":null,"html_url":"https://github.com/umstek/DupKiller","commit_stats":null,"previous_names":[],"tags_count":1,"template":false,"template_full_name":null,"purl":"pkg:github/umstek/DupKiller","repository_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/umstek%2FDupKiller","tags_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/umstek%2FDupKiller/tags","releases_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/umstek%2FDupKiller/releases","manifests_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/umstek%2FDupKiller/manifests","owner_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners/umstek","download_url":"https://codeload.github.com/umstek/DupKiller/tar.gz/refs/heads/master","sbom_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/umstek%2FDupKiller/sbom","scorecard":null,"host":{"name":"GitHub","url":"https://github.com","kind":"github","repositories_count":286080680,"owners_count":35072578,"icon_url":"https://github.com/github.png","version":null,"created_at":"2022-05-30T11:31:42.601Z","updated_at":"2026-05-26T15:22:16.424Z","status":"online","status_checked_at":"2026-07-03T02:00:05.635Z","response_time":110,"last_error":null,"robots_txt_status":"success","robots_txt_updated_at":"2025-07-24T06:49:26.215Z","robots_txt_url":"https://github.com/robots.txt","online":true,"can_crawl_api":true,"host_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub","repositories_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories","repository_names_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repository_names","owners_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners"}},"keywords":["cleaner","duplicate-files","storage"],"created_at":"2024-11-17T15:20:13.219Z","updated_at":"2026-07-03T04:34:11.852Z","avatar_url":"https://github.com/umstek.png","language":"C#","funding_links":[],"categories":[],"sub_categories":[],"readme":"# DupKiller\nSlow, but more reliable duplicate files cleaner. \n\nA perfect duplicate files finder would have to compare the content of each file with the \ncontent of everything else. This has a O(n^2) complexity, and since the files can be \nlarge, this is not possible and will take a long time to complete. If a dictionary i.e.: \na hashmap can be created with the content as keys and paths as values, we can identify \nduplicate files quickly. But this is neither a viable option because all the files will \nhave to be stored in memory and a dictionary will not work properly with that. \nMost of the current software use file size and extension to find the duplicate files but \nthis can be inaccurate in various instances e.g.: uncompressed images that are the same \nsize. \nSo, the best option is to consider multiple factors, and allow user to select whether to \nuse them. These factors should include a hash function. If the user suspects a hash \ncollision, there should be a way to raw compare the files. Since we have filtered \npossible duplicates by various means, such incident can rarely occur. \n\nGroup by extension (optional default yes), file name (optional default no), \nfile size, shorter hash (MD5), longer hash (SHA512)\n","project_url":"https://awesome.ecosyste.ms/api/v1/projects/github.com%2Fumstek%2Fdupkiller","html_url":"https://awesome.ecosyste.ms/projects/github.com%2Fumstek%2Fdupkiller","lists_url":"https://awesome.ecosyste.ms/api/v1/projects/github.com%2Fumstek%2Fdupkiller/lists"}