Remove duplicates values in array Google Apps Script

Question

I would like to know how to remove duplicates values from 2 arrays, combined into one main array.

This values must NOT be removed from the sheets or document, just i

Answer 1

The array you are working on is not a 2D array anymore since you extracted the fields before sorting... so you can use a very simple duplicate removal function as shown below with an example and some added Logger.log to see how it works.

function test(){
  var array = ['aa','bb','cc','aa','dd','cc']
  Logger.log(removeDups(array));
}

function removeDups(array) {
  var outArray = [];
  array.sort():
  outArray.push(array[0]);
  for(var n in array){
    Logger.log(outArray[outArray.length-1]+'  =  '+array[n]+' ?');
    if(outArray[outArray.length-1]!=array[n]){
      outArray.push(array[n]);
    }
  }
  return outArray;
}

in your code this would replace the line

var uniqueData = allData.sort();

that would become :

var uniqueData = removeDups(allData);

---

EDIT :

If letter case is an issue, you can modify this code to ignore it. You should change the condition and the sort function so that they both ignore the case in your names but preferably keep the original letter case.

This could be achieved with the code below :

function test(){
  var array = ['aa','bb','Cc','AA','dd','CC'];// an example with Upper and Lower case
  Logger.log(removeDups(array));
}

function removeDups(array) {
  var outArray = [];
  array.sort(lowerCase);
  function lowerCase(a,b){
    return a.toLowerCase()>b.toLowerCase() ? 1 : -1;// sort function that does not "see" letter case
  }
  outArray.push(array[0]);
  for(var n in array){
    Logger.log(outArray[outArray.length-1]+'  =  '+array[n]+' ?');
    if(outArray[outArray.length-1].toLowerCase()!=array[n].toLowerCase()){
      outArray.push(array[n]);
    }
  }
  return outArray;
}

Logger result :

---

EDIT 2 :

Here is another version that keeps only unique values (I didn't understand correctly your request in the first version as it kept one element from the duplicates...)

I simply added an else if condition to remove the elements that were part of a group of duplicates.(I kept the case insensitive version but you can remove it easily)

function test(){
  var array = ['aa','dd','hh','aa','bb','Cc','cc','cc','bb','nn','bb','AA','dd','CC'];// an example with Upper and Lower case
  Logger.log('original array = '+array);
  Logger.log('unique result = '+removeDups2(array));
}


function removeDups2(array) {
  var uniqueArray = []
  array.sort(lowerCase);
  function lowerCase(a,b){
    return a.toLowerCase()>b.toLowerCase() ? 1 : -1;// sort function that does not "see" letter case
  }
  var temp = array[0];
  for(var n=1 ;n<array.length ; n++){
    Logger.log(temp+'  =  '+array[n]+' ?');
    if(temp.toLowerCase()!=array[n].toLowerCase()){
      uniqueArray.push(array[n]);
      temp = array[n];
    }else if(uniqueArray[uniqueArray.length-1]==temp){
      uniqueArray.pop();// remove it from result if one of the duplicate values
    }
  }
  return uniqueArray;
}

Logger result new code :

Answer 2

In your code you are not doing anything to filter the duplicate values. This line will just sort the data and won't give you unique data.

var uniqueData = allData.sort(); 

You can do something like this on your merged array, after you 'installing' 2DArray lib: https://sites.google.com/site/scriptsexamples/custom-methods/2d-arrays-library

var uniqueData = unique(allData);

Another option is to create a loop and check for duplicate values, but you should remember to transform all the values of the string to lowercase before you do these matches.