$batchMaxSize = 9.5MB
$allFiles = Get-ChildItem 'fileLocation\*.csv'
$batchNumber = 0;
$batchMagnitude = $allFiles.Count.ToString().Length
# group the files together until there's none left
while ($allFiles.Count -gt 0) {
$batchNumber++;
$batchSuffix = $batchNumber.ToString().PadLeft($batchMagnitude,"0");
$totalSize = 0;
# add the file sizes together, keeping a running total until it exceeds the max
$batchFiles = $allFiles | where {$totalSize += $_.Length; $totalSize -lt $batchMaxSize};
# combine the files in memory - don't need to deal with headers
# this step could cause memory issues with extremely large files
# but then you probably wouldn't be batching them ;)
$comboFile = $batchFiles | Foreach-Object {
Import-Csv -Path $_;
}
$comboFile | Export-Csv "batched\batch-$batchSuffix.csv" -Encoding ASCII -NoTypeInformation
# removes the files we just batched together from the list of all files
$allFiles = (Compare-Object $allFiles $batchFiles).InputObject
}