⚡ PowerShell 5.1+

Find Duplicate Files

Recursively scans a folder tree for duplicate files using size pre-filtering plus SHA256 hashing, with an optional safe-delete mode.

By WindowsScripting.com · 2.5 KB
Find-DuplicateFiles.ps1
<#
.SYNOPSIS
    Finds duplicate files in a folder tree based on content hash.

.DESCRIPTION
    Recursively scans -Path, groups files by size first (cheap filter),
    then hashes same-size files with SHA256 to confirm true duplicates.
    Prints each duplicate group and, with -DeleteDuplicates, keeps the
    first file in each group and removes the rest.

.PARAMETER Path
    Root folder to scan.

.PARAMETER DeleteDuplicates
    If specified, deletes all but the first file found in each duplicate group.
    Without this switch the script only reports duplicates (dry run).

.EXAMPLE
    .\Find-DuplicateFiles.ps1 -Path "C:\Users\me\Pictures"

.EXAMPLE
    .\Find-DuplicateFiles.ps1 -Path "C:\Users\me\Downloads" -DeleteDuplicates
#>

[CmdletBinding(SupportsShouldProcess)]
param(
    [Parameter(Mandatory)]
    [ValidateScript({ Test-Path $_ -PathType Container })]
    [string]$Path,

    [switch]$DeleteDuplicates
)

Write-Host "Scanning '$Path' ..." -ForegroundColor Cyan
$allFiles = Get-ChildItem -Path $Path -Recurse -File -ErrorAction SilentlyContinue

$bySize = $allFiles | Group-Object Length | Where-Object { $_.Count -gt 1 }

$duplicateGroups = @()
foreach ($sizeGroup in $bySize) {
    $hashed = $sizeGroup.Group | ForEach-Object {
        [PSCustomObject]@{
            File = $_
            Hash = (Get-FileHash -Path $_.FullName -Algorithm SHA256).Hash
        }
    }
    $byHash = $hashed | Group-Object Hash | Where-Object { $_.Count -gt 1 }
    $duplicateGroups += $byHash
}

if (-not $duplicateGroups) {
    Write-Host "No duplicates found." -ForegroundColor Green
    return
}

$totalWasted = 0
foreach ($group in $duplicateGroups) {
    $files = $group.Group.File
    Write-Host "`nDuplicate set (SHA256 $($group.Name.Substring(0,12))...):" -ForegroundColor Yellow
    $files | ForEach-Object { Write-Host "  $($_.FullName)  ($([math]::Round($_.Length/1KB,1)) KB)" }

    $keep    = $files | Select-Object -First 1
    $remove  = $files | Select-Object -Skip 1
    $totalWasted += ($remove | Measure-Object Length -Sum).Sum

    if ($DeleteDuplicates) {
        foreach ($f in $remove) {
            if ($PSCmdlet.ShouldProcess($f.FullName, "Delete duplicate")) {
                Remove-Item $f.FullName -Force
                Write-Host "  Deleted: $($f.FullName)" -ForegroundColor DarkYellow
            }
        }
    }
}

Write-Host "`nTotal space that can be reclaimed: $([math]::Round($totalWasted/1MB,2)) MB" -ForegroundColor Cyan
if (-not $DeleteDuplicates) {
    Write-Host "Run again with -DeleteDuplicates to remove the extra copies." -ForegroundColor DarkGray
}