/**
 * 删除文件中的重复行
 *
 * @param     string
 * @param     bool
 * @return    string
 * @arrange (512.笔记) jb51.cc
 **/
function RemoveDuplicatedLines($Filepath,$IgnoreCase=false,$NewLine="\n"){
    if (!file_exists($Filepath)){
        $ErrorMsg  = 'RemoveDuplicatedLines error: ';
        $ErrorMsg .= 'The given file ' . $Filepath . ' does not exist!';
        die($ErrorMsg);
    }
    $Content = file_get_contents($Filepath);
    $Content = RemoveDuplicatedLinesByString($Content,$IgnoreCase,$NewLine);

    if (!is_writeable($Filepath)){
        $ErrorMsg  = 'RemoveDuplicatedLines error: ';
        $ErrorMsg .= 'The given file ' . $Filepath . ' is not writeable!';    
        die($ErrorMsg);
    }
 
    $FileResource = fopen($Filepath,'w+');      
    fwrite($FileResource,$Content);        
    fclose($FileResource);   
}
 
 
/**
 * RemoveDuplicatedLinesByString
 * This function removes all duplicated lines of the given string.
 *
 * @param     string
 * @param     bool
 * @return    string
 */
function RemoveDuplicatedLinesByString($Lines,$NewLine="\n"){
    if (is_array($Lines))
        $Lines = implode($NewLine,$Lines);
    $Lines = explode($NewLine,$Lines);
    $LineArray = array();
    $Duplicates = 0;
    // Go trough all lines of the given file
    for ($Line=0; $Line < count($Lines); $Line++){
        // Trim whitespace for the current line
        $CurrentLine = trim($Lines[$Line]);
        // Skip empty lines
        if ($CurrentLine == '')
            continue;
        // Use the line contents as array key
        $LineKey = $CurrentLine;
        if ($IgnoreCase)
            $LineKey = strtolower($LineKey)
        // Check if the array key already exists,// if not add it otherwise increase the counter
        if (!isset($LineArray[$LineKey]))
            $LineArray[$LineKey] = $CurrentLine;        
        else                
            $Duplicates++;
    }
    // Sort the array
    asort($LineArray);
    // Return how many lines got removed
    return implode($NewLine,array_values($LineArray));    
}
//使用范例：
// Example 1
// Removes all duplicated lines of the file definied in the first parameter.
$RemovedLinesCount = RemoveDuplicatedLines('test.txt');
print "Removed $RemovedLinesCount duplicate lines from the test.txt file.";
 
// Example 2 (Ignore case)
// Same as above,just ignores the line case.
RemoveDuplicatedLines('test.txt',true);
 
// Example 3 (Custom new line character)
// By using the 3rd parameter you can define which character
// should be used as new line indicator. In this case
// the example file looks like 'foo;bar;foo;foo' and will
// be replaced with 'foo;bar' 
RemoveDuplicatedLines('test.txt',false,';');


/***   代码来自编程之家 jb51.cc（jb51.cc）   ***/