Sort and uniq a big file

#!/bin/bash
lines=$(wc -l $1 | sed 's/ .*//g')
lines_per_file=`expr $lines / $3`
split -d -l $lines_per_file $1 __part_$1

for file in __part_*
do
{
  echo $file" is sorting"
  sort $file > sort_$file
  rm -rf $file
} &
done
wait

for file in sort_*;do
  sort -smu $file >> $2
  echo $file" merged into "$2
  rm -rf $file
done

rm -f __part_*
rm -f sort_*

duoan / notes

Sort and uniq a big file #28