Find duplicate files

<#
Find files that are exact copies of each other (same content, whatever the name),
keep one of each and move the extra copies into a "Duplicates (review)" folder.
It only shows what it would do unless you add -Apply, and it never deletes
anything: you look through the review folder and delete it yourself.

  powershell -ExecutionPolicy Bypass -File .\find-duplicates.ps1                      Downloads, preview only
  powershell -ExecutionPolicy Bypass -File .\find-duplicates.ps1 -Folder "D:\Photos" -Recurse
  powershell -ExecutionPolicy Bypass -File .\find-duplicates.ps1 -Apply               move the extra copies

Free from Voypelo Solutions, voypelo.co.za. MIT licence.
#>
param(
    [string]$Folder = (Join-Path $env:USERPROFILE 'Downloads'),
    [switch]$Recurse,
    [switch]$Apply
)

$reviewName = 'Duplicates (review)'
if (-not (Test-Path -LiteralPath $Folder -PathType Container)) {
    Write-Error "Expected a folder at $Folder, found nothing there"
    exit 1
}
$review = Join-Path $Folder $reviewName

# Files already parked for review, and files still downloading, are left out.
$files = Get-ChildItem -LiteralPath $Folder -File -Recurse:$Recurse -ErrorAction SilentlyContinue |
    Where-Object { $_.FullName -notlike "$review*" -and $_.Length -gt 0 -and @('.crdownload', '.part', '.tmp') -notcontains $_.Extension.ToLower() }

# Only files of exactly the same size can be copies, so hashing is limited to those.
$groups = $files | Group-Object Length | Where-Object Count -gt 1 | ForEach-Object {
    $_.Group | Group-Object { (Get-FileHash -LiteralPath $_.FullName -Algorithm SHA256).Hash } | Where-Object Count -gt 1
}

$moved = 0
$bytes = 0
foreach ($g in $groups) {
    # Keep the original: a plain name beats "report (1).pdf" or "report - Copy.pdf", then the oldest file.
    $sorted = $g.Group | Sort-Object @{ Expression = { $_.BaseName -match '\(\d+\)$|\bcopy\b' } }, LastWriteTime, FullName
    $keep = $sorted[0]
    Write-Output ('keep        {0}' -f $keep.FullName.Substring($Folder.Length).TrimStart('\'))
    foreach ($extra in ($sorted | Select-Object -Skip 1)) {
        $dest = Join-Path $review $extra.Name
        # Never overwrite: a clash in the review folder gets a number added.
        $n = 1
        while (Test-Path -LiteralPath $dest) {
            $dest = Join-Path $review ('{0} ({1}){2}' -f $extra.BaseName, $n, $extra.Extension)
            $n++
        }
        $rel = $extra.FullName.Substring($Folder.Length).TrimStart('\')
        if ($Apply) {
            New-Item -ItemType Directory -Path $review -Force | Out-Null
            Move-Item -LiteralPath $extra.FullName -Destination $dest
            Write-Output ('  moved     {0}' -f $rel)
        } else {
            Write-Output ('  copy of it, would move  {0}' -f $rel)
        }
        $moved++
        $bytes += $extra.Length
    }
}

$size = if ($bytes -ge 1MB) { '{0:N1} MB' -f ($bytes / 1MB) } else { '{0:N0} KB' -f [math]::Ceiling($bytes / 1KB) }
if ($moved -eq 0) {
    Write-Output 'No duplicate files found.'
} elseif ($Apply) {
    Write-Output ('{0} extra cop(ies) moved to "{1}" ({2}). Look through it, then delete it yourself.' -f $moved, $reviewName, $size)
} else {
    Write-Output ('{0} extra cop(ies) found ({1}). Run again with -Apply to move them to "{2}".' -f $moved, $size, $reviewName)
}