How to merge two arrays in JavaScript and de-duplicate items

Question

I have two JavaScript arrays:

var array1 = ["Vijendra","Singh"];
var array2 = ["Singh", "Shakya"];

I want the output to be:

var array3 = ["Vijendra","Singh","Shakya"];

The output array should have repeated words removed.

How do I merge two arrays in JavaScript so that I get only the unique items from each array in the same order they were inserted into the original arrays?

Answer 1

For n arrays, you can get the union like so.

function union(arrays) {
    return new Set(arrays.flat()).keys();
};

Answer 2

Built a tester to check just how fast some of the performance oriented answers are. Feel free to add some more. So far, Set is both the simplest and fastest option (by bigger margins as the number of records increases), at least with simple Number types.

const records = 10000, //max records per array
  max_int = 100, //max integer value per array
  dup_rate = .5; //rate of duplication
let perf = {}, //performance logger,
  ts = 0,
  te = 0,
  array1 = [], //init arrays
  array2 = [],
  array1b = [],
  array2b = [],
  a = [];

//populate randomized arrays
for (let i = 0; i < records; i++) {
  let r = Math.random(),
    n = r * max_int;
  if (Math.random() < .5) {
    array1.push(n);
    r < dup_rate && array2.push(n);
  } else {
    array2.push(n);
    r < dup_rate && array1.push(n);
  }
}
//simple deep copies short of rfdc, in case someone wants to test with more complex data types
array1b = JSON.parse(JSON.stringify(array1));
array2b = JSON.parse(JSON.stringify(array2));
console.log('Records in Array 1:', array1.length, array1b.length);
console.log('Records in Array 2:', array2.length, array2b.length);

//test method 1 (jsperf per @Pitouli)
ts = performance.now();
for (let i = 0; i < array2.length; i++)
  if (array1.indexOf(array2[i]) === -1)
    array1.push(array2[i]); //modifies array1
te = performance.now();
perf.m1 = te - ts;
console.log('Method 1 merged', array1.length, 'records in:', perf.m1);
array1 = JSON.parse(JSON.stringify(array1b)); //reset array1

//test method 2 (classic forEach)
ts = performance.now();
array2.forEach(v => array1.includes(v) ? null : array1.push(v)); //modifies array1
te = performance.now();
perf.m2 = te - ts;
console.log('Method 2 merged', array1.length, 'records in:', perf.m2);

//test method 3 (Simplest native option)
ts = performance.now();
a = [...new Set([...array1, ...array2])]; //does not modify source arrays
te = performance.now();
perf.m3 = te - ts;
console.log('Method 3 merged', a.length, 'records in:', perf.m3);

//test method 4 (Selected Answer)
ts = performance.now();
a = array1.concat(array2); //does not modify source arrays
for (let i = 0; i < a.length; ++i) {
  for (let j = i + 1; j < a.length; ++j) {
    if (a[i] === a[j])
      a.splice(j--, 1);
  }
}
te = performance.now();
perf.m4 = te - ts;
console.log('Method 4 merged', a.length, 'records in:', perf.m4);

//test method 5 (@Kamil Kielczewski)
ts = performance.now();

function K(arr1, arr2) {
  let r = [],
    h = {};

  while (arr1.length) {
    let e = arr1.shift(); //modifies array1
    if (!h[e]) h[e] = 1 && r.push(e);
  }

  while (arr2.length) {
    let e = arr2.shift(); //modifies array2
    if (!h[e]) h[e] = 1 && r.push(e);
  }

  return r;
}
a = K(array1, array2);
te = performance.now();
perf.m5 = te - ts;
console.log('Method 5 merged', a.length, 'records in:', perf.m4);
array1 = JSON.parse(JSON.stringify(array1b)); //reset array1
array2 = JSON.parse(JSON.stringify(array2b)); //reset array2


for (let i = 1; i < 6; i++) {
  console.log('Method:', i, 'speed is', (perf['m' + i] / perf.m1 * 100).toFixed(2), '% of Method 1');
}

Answer 3

To offer something simpler and more elegant, in this day and age, using an existing library:

import {pipe, concat, distinct} from 'iter-ops';

// our inputs:
const array1 = ['Vijendra', 'Singh'];
const array2 = ['Singh', 'Shakya'];

const i = pipe(
    array1,
    concat(array2), // adding array
    distinct() // making it unique
);

console.log([...i]); //=> ['Vijendra', 'Singh', 'Shakya']

It is both high performance, as we are iterating only once, and the code is very easy to read.

P.S. I'm the author of iter-ops.

Answer 4

Using Array.prototype.flat():

const input = [
  [1, 2, 3, 1],
  [101, 2, 1, 10],
  [2, 1]
];

const union = (arr) => {
  return [  ...new Set( arr.flat() )  ];
}

console.log('output', union(input));

Answer 5

We can concat method use to add two or more array

array1.concat(array2, array3, ..., arrayX)

// unique remove duplicates of array

console.log([...new Set([1, 2, 4, 4, 3])]); // [1, 2, 4, 3]

Answer 6

Here is my solution https://gist.github.com/4692150 with deep equals and easy to use result:

function merge_arrays(arr1,arr2)
{
   ... 
   return {first:firstPart,common:commonString,second:secondPart,full:finalString}; 
}

console.log(merge_arrays(
[
[1,"10:55"] ,
[2,"10:55"] ,
[3,"10:55"]
],[
[3,"10:55"] ,
[4,"10:55"] ,
[5,"10:55"]
]).second);

result:
[
[4,"10:55"] ,
[5,"10:55"]
]

Answer 7

Just wrote before for the same reason (works with any amount of arrays):

/**
 * Returns with the union of the given arrays.
 *
 * @param Any amount of arrays to be united.
 * @returns {array} The union array.
 */
function uniteArrays()
{
    var union = [];
    for (var argumentIndex = 0; argumentIndex < arguments.length; argumentIndex++)
    {
        eachArgument = arguments[argumentIndex];
        if (typeof eachArgument !== 'array')
        {
            eachArray = eachArgument;
            for (var index = 0; index < eachArray.length; index++)
            {
                eachValue = eachArray[index];
                if (arrayHasValue(union, eachValue) == false)
                union.push(eachValue);
            }
        }
    }

    return union;
}    

function arrayHasValue(array, value)
{ return array.indexOf(value) != -1; }

Answer 8

Array.prototype.pushUnique = function(values)
{
    for (var i=0; i < values.length; i++)
        if (this.indexOf(values[i]) == -1)
            this.push(values[i]);
};

Try:

var array1 = ["Vijendra","Singh"];
var array2 = ["Singh", "Shakya"];
array1.pushUnique(array2);
alert(array1.toString());  // Output: Vijendra,Singh,Shakya

Answer 9

If, like me, you need to support older browsers, this works with IE6+

function es3Merge(a, b) {
    var hash = {},
        i = (a = a.slice(0)).length,
        e;

    while (i--) {
        hash[a[i]] = 1;
    }

    for (i = 0; i < b.length; i++) {
        hash[e = b[i]] || a.push(e);
    }

    return a;
};

http://jsperf.com/merge-two-arrays-keeping-only-unique-values/22

Answer 10

This is simple and can be done in one line with jQuery:

var arr1 = ['Vijendra', 'Singh'], arr2 =['Singh', 'Shakya'];

$.unique(arr1.concat(arr2))//one line

["Vijendra", "Singh", "Shakya"]

Answer 11

I came across this post when trying to do the same thing, but I wanted to try something different. I just made up the function below. I also had another variable, 'compareKeys', (array of keys) for doing shallow object comparison. I'm going to probably change it to a function in the future.

Anyway, I didn't include that part, because it doesn't apply to the question. I also put my code into the jsperf going around. Edited: I fixed my entry in jsperf. My function gets about 99k ops/sec compared to 140k.

To the code: I first make an array of the available indices and then eliminate them by iterating over the first array. Finally, I push in the 'left-overs' by using the trimmed down array of indices that didn't match between the two arrays.

http://jsperf.com/merge-two-arrays-keeping-only-unique-values/26

function indiceMerge(a1, a2) {
    var ai = [];
    for (var x = 0; x < a2.length; x++) {
        ai.push(x)
    };

    for (var x = 0; x < a1.length; x++) {
        for (var y = 0; y < ai.length; y++) {
            if (a1[x] === a2[ai[y]]) {
                ai.splice(y, 1);
                y--;
            }
        }
    }

    for (var x = 0; x < ai.length; x++) {
        a1.push(a2[ai[x]]);
    }

    return a1;
}

Answer 12

Usage: https://gist.github.com/samad-aghaei/7250ffb74ed80732debb1cbb14d2bfb0

var _uniqueMerge = function(opts, _ref){
    for(var key in _ref)
        if (_ref && _ref[key] && _ref[key].constructor && _ref[key].constructor === Object)
          _ref[key] = _uniqueMerge((opts ? opts[key] : null), _ref[key] );
        else if(opts && opts.hasOwnProperty(key))
          _ref[key] = opts[key];
        else _ref[key] = _ref[key][1];
    return _ref;
}

Answer 13

The best and simplest way to do that is using the function "some()" of JavaScript that returns true or false indicating if the array contains the object's element. You can make this:

var array1 = ["Vijendra","Singh"]; 
var array2 = ["Singh", "Shakya"];

var array3 = array1;

array2.forEach(function(elementArray2){
    var isEquals = array1.some(function(elementArray1){
        return elementArray1 === elementArray2;
    })
    if(!isEquals){
        array3.push(elementArray2);
    }
});
console.log(array3);

The results:

["Vijendra", "Singh", "Shakya"]

ss you wish... without duplicate it...

Answer 14

var a = [1,2,3]
var b = [1,2,4,5]

I like one liners. This will push distinct b elements to a

b.forEach(item => a.includes(item) ? null : a.push(item));

And another version that will not modify a

var c = a.slice();
b.forEach(item => c.includes(item) ? null : c.push(item));

Answer 15

If you're purely using underscore.js, it doesn't have unionWith, unionBy

you can try out : _.uniq(_.union(arr1, arr2), (obj) => obj.key) ( key is the key param of each object ) this should help to get unique after union of both arrays.

Answer 16

I think this works faster.

removeDup = a => {

    for (let i = a.length - 1; i >= 0; i--) {
        for (let j = i-1; j >= 0; j--) {
            if (a[i] === a[j])
                a.splice(j--, 1);
        }
    }

    return a;
}

Answer 17

   //1.merge two array into one array

   var arr1 = [0, 1, 2, 4];
   var arr2 = [4, 5, 6];

   //for merge array we use "Array.concat"

   let combineArray = arr1.concat(arr2); //output

   alert(combineArray); //now out put is 0,1,2,4,4,5,6 but 4 reapeat

   //2.same thing with "Spread Syntex"

   let spreadArray = [...arr1, ...arr2];

   alert(spreadArray);  //now out put is 0,1,2,4,4,5,6 but 4 reapete


   /*
       if we need remove duplicate element method use are
       1.Using set
       2.using .filter
       3.using .reduce
   */

Answer 18

Not Performant if you have extremely large lists, and this isnt for merging since many solutions already have been documented, but i solved my problems with this solution (since most solutions of array filtering apply to simple arrays)

const uniqueVehiclesServiced = 
  invoice.services.sort().filter(function(item, pos, ary) {
    const firstIndex = invoice.services.findIndex((el, i, arr) => el.product.vin === item.product.vin)

  return !pos || firstIndex == pos;
});

Answer 19

I have a similar request but it is with Id of the elements in the array.

And, here is the way I do the deduplication.

It is simple, easy to maintain, and good to use.

// Vijendra's Id = Id_0
// Singh's Id = Id_1
// Shakya's Id = Id_2

let item0 = { 'Id': 'Id_0', 'value': 'Vijendra' };
let item1 = { 'Id': 'Id_1', 'value': 'Singh' };
let item2 = { 'Id': 'Id_2', 'value': 'Shakya' };

let array = [];

array = [ item0, item1, item1, item2 ];

let obj = {};
array.forEach(item => {
    obj[item.Id] = item;
});

let deduplicatedArray = [];
let deduplicatedArrayOnlyValues = [];
for(let [index, item] of Object.values(obj).entries()){
    deduplicatedArray = [ ...deduplicatedArray, item ];
    deduplicatedArrayOnlyValues = [ ...deduplicatedArrayOnlyValues , item.value ];
};
    
console.log( JSON.stringify(array) );
console.log( JSON.stringify(deduplicatedArray) );
console.log( JSON.stringify(deduplicatedArrayOnlyValues ) );

The console log

[{"recordId":"Id_0","value":"Vijendra"},{"recordId":"Id_1","value":"Singh"},{"recordId":"Id_1","value":"Singh"},{"recordId":"Id_2","value":"Shakya"}]

[{"recordId":"Id_0","value":"Vijendra"},{"recordId":"Id_1","value":"Singh"},{"recordId":"Id_2","value":"Shakya"}]

["Vijendra","Singh","Shakya"]

Answer 20

return an array of unique objects from arrays...

function uniqueObjectArray(baseArray, mergeArray) {
    // we can't compare unique objects within an array ~es6...
    // ...e.g. concat/destructure/Set()
    // so we'll create a mapping of: item.id* for each -> item
    const uniqueMap = new Map()
    const uniqueArray = []

    // hash array items by id*
    baseArray.forEach(item => !uniqueMap.has(item.id) && uniqueMap.set(item.id, item))
    mergeArray.forEach(item => !uniqueMap.has(item.id) && uniqueMap.set(item.id, item))

    // hash -> array
    uniqueMap.forEach(item => uniqueArray.push(item))
    return uniqueArray
}

Answer 21

My quick take

const arePropsEqualDefault = (oldProps, newProps) => {
  const uniqueKeys = Object.values({...Object.keys(oldProps), ...Object.keys(newProps)});
  for (const key in uniqueKeys) {
    console.log(Object.is(oldProps[key], newProps[key]))
    if (!Object.is(oldProps[key], newProps[key])) return false;
  }
  return true;
}

Answer 22

This is the function I use when I need to merge, (or return the union of) two arrays.

var union = function (a, b) {
  for (var i = 0; i < b.length; i++)
    if (a.indexOf(b[i]) === -1)
      a.push(b[i]);
  return a;
};

var a = [1, 2, 3, 'a', 'b', 'c'];
var b = [2, 3, 4, 'b', 'c', 'd'];

a = union(a, b);
//> [1, 2, 3, "a", "b", "c", 4, "d"]

var array1 = ["Vijendra", "Singh"];
var array2 = ["Singh", "Shakya"];

var array3 = union(array1, array2);
//> ["Vijendra", "Singh", "Shakya"]

Answer 23

function set(a, b) {
  return a.concat(b).filter(function(x,i,c) { return c.indexOf(x) == i; });
}

Answer 24

Better option for large inputs will be to sort arrays. Then merge them.

function sortFunction(a, b) {
        return a - b;
}

arr1.sort(sortFunction);
arr2.sort(sortFunction);

function mergeDedup(arr1, arr2) {

    var i = 0, j = 0, result = [];
    while (i < arr1.length && j < arr2.length) {

        if (arr1[i] < arr2[j]) {
            writeIfNotSameAsBefore(result, arr1[i]);
            i++;
        }
        else if (arr1[i] > arr2[j]) {
            writeIfNotSameAsBefore(result, arr2[j]);
            j++;
        }
        else {
            writeIfNotSameAsBefore(result, arr1[i]);
            i++;
            j++;
        }

    }

    while (i < arr1.length) {
        writeIfNotSameAsBefore(result, arr1[i]);
        i++;
    }

    while (j < arr2.length) {
        writeIfNotSameAsBefore(result, arr2[j]);
        j++;
    }
    return result;
}

function writeIfNotSameAsBefore(arr, item) {
    if (arr[arr.length - 1] !== item) {
        arr[arr.length] = item;
    }
    return arr.length;
}

Sorting will take O(nlogn + mlogm), where n and m are length of the arrays, and O(x) for merging, where x = Max(n, m);

Answer 25

You can merge the results and filter the duplicates:

let combinedItems = [];

// items is an Array of arrays: [[1,2,3],[1,5,6],...]    
items.forEach(currItems => {
    if (currItems && currItems.length > 0) {
        combinedItems = combinedItems.concat(currItems);
    }
});

let noDuplicateItems = combinedItems.filter((item, index) => {
    return !combinedItems.includes(item, index + 1);
});

Answer 26

/**
 * De-duplicate an array keeping only unique values.
 * Use hash table (js object) to filter-out duplicates.
 * The order of array elements is maintained.
 * This algorithm is particularly efficient for large arrays (linear time).
 */
function arrayUniqueFast(arr) {
	var seen = {};
	var result = [];
	var i, len = arr.length;
	for (i = 0; i < len; i++) {
		var item = arr[i];
		// hash table lookup
		if (!seen[item]) {
			result.push(item);
			seen[item] = true;
		}
	}
	return result;
}

///// test
var array1 = ["Vijendra", "Singh"];
var array2 = ["Singh", "Shakya"];

var result = arrayUniqueFast(array1.concat(array2));
document.write('<br>result: ' + result);

For other methods to dedup an array, please see my benchmarks at: https://jsperf.com/de-duplicate-an-array-keeping-only-unique-values

Answer 27

This is fast, collates any number of arrays, and works with both numbers and strings.

function collate(a){ // Pass an array of arrays to collate into one array
    var h = { n: {}, s: {} };
    for (var i=0; i < a.length; i++) for (var j=0; j < a[i].length; j++)
        (typeof a[i][j] === "number" ? h.n[a[i][j]] = true : h.s[a[i][j]] = true);
    var b = Object.keys(h.n);
    for (var i=0; i< b.length; i++)
        b[i]=Number(b[i]);
    return b.concat(Object.keys(h.s));
}

> a = [ [1,2,3], [3,4,5], [1,5,6], ["spoon", "fork", "5"] ]
> collate( a )

[1, 2, 3, 4, 5, 6, "5", "spoon", "fork"]

If you don't need to distinguish between 5 and "5", then

function collate(a){
    var h = {};
    for (i=0; i < a.length; i++) for (var j=0; j < a[i].length; j++)
        h[a[i][j]] = typeof a[i][j] === "number";
    for (i=0, b=Object.keys(h); i< b.length; i++)
        if (h[b[i]])
            b[i]=Number(b[i]);
    return b;
}
[1, 2, 3, 4, "5", 6, "spoon", "fork"]

will do.

And if you don't mind (or would prefer) all values ending up as strings anyway then just this:

function collate(a){
    var h = {};
    for (var i=0; i < a.length; i++)
        for (var j=0; j < a[i].length; j++)
            h[a[i][j]] = true;
    return Object.keys(h)
}
["1", "2", "3", "4", "5", "6", "spoon", "fork"]

If you don't actually need an array, but just want to collect the unique values and iterate over them, then (in most browsers (and node.js)):

h = new Map();
for (i=0; i < a.length; i++)
    for (var j=0; j < a[i].length; j++)
        h.set(a[i][j]);

It might be preferable.

Answer 28

I learned a cheeky little way to concatenate two arrays with the spread operator:

var array1 = ['tom', 'dick', 'harry'];
var array2 = ['martin', 'ricky'];

array1.push(...array2);

The "..." spread operator splits the following array into individual items and then push can handle them as separate arguments.

Answer 29

Given two sorted arrays of simple types without duplicates, this will merge them in O(n) time, and the output will be also sorted.

function merge(a, b) {
    let i=0;
    let j=0;
    let c = [];
    for (;;) {
        if (i == a.length) {
            if (j == b.length) return c;
            c.push(b[j++]);
        } else if (j == b.length || a[i] < b[j]) {
            c.push(a[i++]);
        } else {
            if (a[i] == b[j]) ++i;   // skip duplicates
            c.push(b[j++]);
        }
    }
}

Answer 30

const merge(...args)=>(new Set([].concat(...args)))