用moment.js吧
參考文件，你查查看 addWeekDays() 應該就是你需要的功能。
https://momentjs.com/docs/

一週用五天去算
編號：把週一當作 0，週二當作 1，週五當作 4

分別把 Days「 對 5 取餘數」、「對 5 取整數除法」
然後再去計算處理

打包成函式

function addByWorkDay(startDate, nDays)
{
    let arr=startDate.split('-').map(x=>parseInt(x,10));
    let date=new Date(arr[0], arr[1]-1, arr[2]);

    //計算需增加 nWeeks 週又 modDay 天
    let modDay=nDays%5;
    let nWeeks=(nDays-modDay)/5;

    //計算要加幾天
    let wkBegin=date.getDay();
    let dayOffset=nWeeks*7+modDay+((wkBegin+6)%7+modDay>4?2:0);

    date.setDate(date.getDate()+dayOffset);
    return [date.getFullYear(), date.getMonth()+1, date.getDate()].map(x=>x<10?'0'+x:x).join('-');
}

測試

let start='2021-01-04';
for(let i=0;i<16;++i) {
    console.log(`${start} + ${i} = ${addByWorkDay(start, i)}`);
}

另一個版本供參考

function addWorkDays(startDate, days) {
    if(isNaN(days)) {
        console.log("Value provided for \"days\" was not a number");
        return
    }
    if(!(startDate instanceof Date)) {
        console.log("Value provided for \"startDate\" was not a Date object");
        return
    }
    // Get the day of the week as a number (0 = Sunday, 1 = Monday, .... 6 = Saturday)
    var dow = startDate.getDay();
    var daysToAdd = parseInt(days);
    // If the current day is Sunday add one day
    if (dow == 0)
        daysToAdd++;
    // If the start date plus the additional days falls on or after the closest Saturday calculate weekends
    if (dow + daysToAdd >= 6) {
        //Subtract days in current working week from work days
        var remainingWorkDays = daysToAdd - (5 - dow);
        //Add current working week's weekend
        daysToAdd += 2;
        if (remainingWorkDays > 5) {
            //Add two days for each working week by calculating how many weeks are included
            daysToAdd += 2 * Math.floor(remainingWorkDays / 5);
            //Exclude final weekend if remainingWorkDays resolves to an exact number of weeks
            if (remainingWorkDays % 5 == 0)
                daysToAdd -= 2;
        }
    }
    startDate.setDate(startDate.getDate() + daysToAdd);
    return startDate;
}

//And use it like so (months are zero based)
var today = new Date(2016, 10, 22);
today = addWorkDays(today, 5);

資料來源

$(document).ready(function(){
    $('#calc').click(function(){
  var d1 = $('#d1').val();
  var d2 = $('#d2').val();
        $('#dif').text(workingDaysBetweenDates(d1,d2));
    });
});
function workingDaysBetweenDates(d0, d1) {
    var startDate = parseDate(d0);
    var endDate = parseDate(d1);    
    //節日陣列，自己改一下 僅列到228補假，其他自行以行事曆列出
    var holidays = ['2021-01-01','2021-02-10','2021-02-11','2021-02-12','2021-02-13','2021-02-14','2021-02-15','2021-02-16', '2018-03-01']; 
    // 結束日一定要大於開始日，否則回傳0
    if (endDate < startDate)
        return 0;

    var z = 0; // 遇到國定假日的日數
    for (i = 0; i < holidays.length; i++)
    {
        var cand = parseDate(holidays[i]);
        var candDay = cand.getDay();

      if (cand >= startDate && cand <= endDate && candDay != 0 && candDay != 6)
      {
        // 如果假日不是星期六日，就
        z++;
      }

    }
    // 計算兩個日期間的總日數
    var millisecondsPerDay = 86400 * 1000; // 一天的總milliseconds微秒數
    startDate.setHours(0,0,0,1);  // 一天的起點
    endDate.setHours(23,59,59,999);  // 終點
    var diff = endDate - startDate;  // 兩天之間的總微秒數
    var days = Math.ceil(diff / millisecondsPerDay);

    // 算出周數weeks
    var weeks = Math.floor(days / 7);
    // 總日數 - (周數*2)=實際工作天(包含國定假日)
    days = days - (weeks * 2);

    var startDay = startDate.getDay();
    var endDay = endDate.getDay();

    // 減去周末
    if (startDay - endDay > 1)         
        days = days - 2;      

    // 如果開始日是星期天，結束日不是星期六
    if (startDay == 0 && endDay != 6)
        days = days - 1  

    // 如果結束日星期六，開始日不是星期天
    if (endDay == 6 && startDay != 0)
        days = days - 1  

    // 減掉國定假日
    return days - z;
}
function parseDate(input) {
    // 轉換日期格式
  var parts = input.match(/(\d+)/g);
  // new Date(year, month [, date [, hours[, minutes[, seconds[, ms]]]]])
  return new Date(parts[0], parts[1]-1, parts[2]); // months are 0-based
}

出勤系統都會遇到同樣的問題，
單單計算假日不可靠，比如說突然颱風天放假，跨月份補上班或跨月份連假，
這些都不是單純扣掉假日(六&日)就能解決的。

這個要用後端資料庫來做才會【有效能】

Date.prototype.addDays = function(days) {
  this.setDate(this.getDate() + days);
  return this;
}

Date.prototype.addWorkDays = function(days) {
  const weekday = this.getDay() == 0? 6: this.getDay() -1
  this.addDays( parseInt(days/5)*2 + days )
  if (weekday >= 4) this.addDays( 6-weekday )
  return this;
}