import { parse } from "https://deno.land/std/flags/mod.ts"; import { walkSync} from "https://deno.land/std/fs/mod.ts"; import { createHash } from "https://deno.land/std/hash/mod.ts"; const MAX_SIZE = Math.pow(2, 29); function getMD5(content) { return createHash("md5").update(content).toString(); } function main () { const { _: folders, delete: deleteDuplicates = false, } = parse(Deno.args, { alias: {'d': 'delete'}}); const folderList = folders.reduce((acc, folder) => { const fileInfo = Deno.statSync(folder); if (fileInfo.isDirectory) { acc.push(folder); } return acc; }, []); const filesToCheck = folderList.reduce((acc, folder) => { for (const entry of walkSync(folder)) { const stat = Deno.statSync(entry.path); if (entry.isFile && !acc.includes(entry.path) && stat.size < MAX_SIZE) { acc.push(entry.path); } } return acc; }, []); const checksumDict = filesToCheck.reduce((acc, file) => { const content = Deno.readTextFileSync(file); const md5 = getMD5(content); if (!acc[md5]) { acc[md5] = []; } else { acc[md5].push(file); } return acc; }, {}) let foundDuplicates = false; for (const [checksum, files] of Object.entries(checksumDict)) { const duplicates = files.length; if (duplicates > 0) { foundDuplicates = true; console.info(`${duplicates} duplicate files found: ${['', ...files].join('\n * ')}`); if (deleteDuplicates) { files.map(file => { console.info(`Deleting file: ${file}`); Deno.removeSync(file); }) } } } if (!foundDuplicates) { console.info(`No duplicates found in ${folderList}`); } } main();