What is the best practice to read date type value?

With cellDates read option, xlsx library tries to convert date-type cell to js Date object. However, it does not seem to respect date1904 property of sheet when constructing js Date object. https://github.com/SheetJS/js-xlsx/issues/126

excel_date.xlsx

const xlsx = require('xlsx');
const ws = xlsx.readFile('./excel_date.xlsx', {cellDates: true});
console.log('date1904:', ws.Workbook.WBProps.date1904);
const firstSheet = ws.Sheets[ws.SheetNames[0]];
console.log(xlsx.utils.sheet_to_json(firstSheet));

The above code with the attached excel file gives the following result:

date1904: true
[ { Date: 2014-12-30T14:59:08.000Z,
    String: 'I am text',
    number: 1 },
  { Date: '2019-01-01', String: 1, number: 2 },
  { Date: 2014-12-30T14:59:08.000Z, String: '3', number: 3 },
  { Date: 2014-12-30T14:59:08.000Z, String: 2, number: 4 } ]

I expected that the generated js Date objects are of ‘2019-01-01’, but they are skewed due to date1904 problem. I converted all js Date values in my program. But I think It would be better that the library do this magical conversion so that users do not need to consider date1904 anymore. Am I missing useful option?

[zoeesilcock]

This thread has ballooned to include a general discussion about handling dates. Could we return to the original issue, namely that readFile doesn’t return correct data? As was described in the original issue, the library correctly identifies that the file is based on 1904 rather than 1900 but the dates in the result don’t reflect that. Surely this isn’t meant to work this way?

[jngbng]

@zoeesilcock There are two more problems. SSF module output, instead of JS native Date, is preferable to represent date when importing excel file.

TL;DR. I am using following workaround code.

// Take following code from xlsx@0.15.1.
// They are private scoped and inaccessible from outside of the library.
const basedate = new Date(1899, 11, 30, 0, 0, 0);
const dnthresh = basedate.getTime() + (new Date().getTimezoneOffset() - basedate.getTimezoneOffset()) * 60000;

const day_ms = 24 * 60 * 60 * 1000;
const days_1462_ms = 1462 * day_ms;

function datenum(v, date1904) {
  let epoch = v.getTime();
  if (date1904) {
    epoch -= days_1462_ms;
  }
  return (epoch - dnthresh) / day_ms;
}

function fixImportedDate(date, is_date1904) {
  // Convert JS Date back to Excel date code and parse them using SSF module.
  const parsed = xlsx.SSF.parse_date_code(datenum(date, false), {date1904: is_date1904});
  return `${parsed.y}-${parsed.m}-${parsed.d}`;
  // or
  // return parsed;
  // or if you want to stick to JS Date,
  // return new Date(parsed.y, parsed.m, parsed.d, parsed.H, parsed.M, parsed.S);
}

function useSSFOutput() {
  const wb = xlsx.readFile('./tz_test_dates.xlsx', {cellDates: true});
  const sheet = wb.Sheets[(wb.SheetNames[0])];
  // original output
  const converted = xlsx.utils.sheet_to_json(sheet, {header: 1, cellDates:true});
  // apply hotfix
  const is_date1904 = wb.Workbook.WBProps.date1904;
  const fixed = converted.map((arr) => arr.map((v) => {
    if (v instanceof Date) {
      return fixImportedDate(v, is_date1904);
    } else {
      return v;
    }
  }));
  console.log(fixed.map(arr => arr.map(v => v.toString())));
}

useSSFOutput();

Run above code with tz_test_dates.xlsx and will get following result:

tz_test_dates.xlsx preview:

2019-01-01 | 1960-01-01 | 1908-01-01 |   |   |   – | – | – | – | – | – 2019-01-01 | 2019-03-01 | 2019-05-01 | 2019-07-01 | 2019-09-01 | 2019-11-01

[ [ '2019-1-1', '1960-1-1', '1908-1-1' ],
  [ '2019-1-1',
    '2019-3-1',
    '2019-5-1',
    '2019-7-1',
    '2019-9-1',
    '2019-11-1' ] ]

Detail

sheet_to_json uses following code to convert Excel date code to JS Date object.

var basedate = new Date(1899, 11, 30, 0, 0, 0); // 2209161600000
var dnthresh = basedate.getTime() + (new Date().getTimezoneOffset() - basedate.getTimezoneOffset()) * 60000;

function numdate(v) {
	var out = new Date();
	out.setTime(v * 24 * 60 * 60 * 1000 + dnthresh);
	return out;
}

issue 1: precision bug. refer to #1470

On some countries, you may lose some time (in Korea, -52 sec when parsing). The problem is that Date.getTimezoneOffset() is not precise enough. (SheetJS/ssf#38)

function showOriginal() {
  const wb = xlsx.readFile('./tz_test_dates.xlsx', {cellDates: true});
  const sheet = wb.Sheets[(wb.SheetNames[0])];
  // original output
  const converted = xlsx.utils.sheet_to_json(sheet, {header: 1, cellDates:true});
  console.log('showOriginal:');
  console.log(converted.map(arr => arr.map(v => v.toString())));
}

showOriginal();

/////////////////////////////////////

function getTimezoneOffsetMS(date) {
  var time = date.getTime();
  var utcTime = Date.UTC(date.getFullYear(),
                         date.getMonth(),
                         date.getDate(),
                         date.getHours(),
                         date.getMinutes(),
                         date.getSeconds(),
                         date.getMilliseconds());
  return time - utcTime;
}

const importBugHotfixDiff = (function () {
  const basedate = new Date(1899, 11, 30, 0, 0, 0);
  const dnthreshAsIs = (new Date().getTimezoneOffset() - basedate.getTimezoneOffset()) * 60000;
  const dnthreshToBe = getTimezoneOffsetMS(new Date()) - getTimezoneOffsetMS(basedate);
  return dnthreshAsIs - dnthreshToBe;
}());

function fixPrecisionLoss(date) {
  return (new Date(date.getTime() - importBugHotfixDiff));
}

function showPrevisionLossHotfix() {
  const wb = xlsx.readFile('./tz_test_dates.xlsx', {cellDates: true});
  const sheet = wb.Sheets[(wb.SheetNames[0])];
  // original output
  const converted = xlsx.utils.sheet_to_json(sheet, {header: 1, cellDates:true});
  // apply hotfix. ignore date1904 problem for now.
  const fixed = converted.map((arr) => arr.map((v) => {
    if (v instanceof Date) {
      return fixPrecisionLoss(v);
    } else {
      return v;
    }
  }));
  console.log('showPrevisionLossHotfix:');
  console.log(fixed.map(arr => arr.map(v => v.toString())));
}

showPrevisionLossHotfix();

Run above code after setting computer’s time zone to Asia/Seoul (UTC+09:00) then will get:

showOriginal:
[ [ 'Mon Dec 31 2018 23:59:08 GMT+0900 (Korean Standard Time)',
    'Thu Dec 31 1959 23:29:08 GMT+0830 (Korean Standard Time)',
    'Tue Dec 31 1907 23:27:00 GMT+0827 (Korean Standard Time)' ],
  [ 'Mon Dec 31 2018 23:59:08 GMT+0900 (Korean Standard Time)',
    'Thu Feb 28 2019 23:59:08 GMT+0900 (Korean Standard Time)',
    'Tue Apr 30 2019 23:59:08 GMT+0900 (Korean Standard Time)',
    'Sun Jun 30 2019 23:59:08 GMT+0900 (Korean Standard Time)',
    'Sat Aug 31 2019 23:59:08 GMT+0900 (Korean Standard Time)',
    'Thu Oct 31 2019 23:59:08 GMT+0900 (Korean Standard Time)' ] ]
showPrevisionLossHotfix:
[ [ 'Tue Jan 01 2019 00:00:00 GMT+0900 (Korean Standard Time)',
    'Thu Dec 31 1959 23:30:00 GMT+0830 (Korean Standard Time)',
    'Tue Dec 31 1907 23:27:52 GMT+0827 (Korean Standard Time)' ],
  [ 'Tue Jan 01 2019 00:00:00 GMT+0900 (Korean Standard Time)',
    'Fri Mar 01 2019 00:00:00 GMT+0900 (Korean Standard Time)',
    'Wed May 01 2019 00:00:00 GMT+0900 (Korean Standard Time)',
    'Mon Jul 01 2019 00:00:00 GMT+0900 (Korean Standard Time)',
    'Sun Sep 01 2019 00:00:00 GMT+0900 (Korean Standard Time)',
    'Fri Nov 01 2019 00:00:00 GMT+0900 (Korean Standard Time)' ] ]

Notice that 52 seconds error has gone, but ‘1960-01-01’ and ‘1908-01-01’ are not correctly parsed. It is due to following issue.

issue2: timezone offset is not constant within one time zone.

Noice that dnthresh depends on the timezone offset of CURRENT TIME. But on some countries, timezone offset changes (or have changed) over time. In Korea, it is GMT+09:00 now, but it was GMT+08:30 in 1960 and GMT+08:27 in 1908. In Los Angeles in US, it is GMT-08:00 in January and GMT-07:00 in October due to summer time. For these countries, dnthresh should not be constant and we should consider time zone change. SSF module which is timezone-agnostic rescues us.

// --------------------------------------------------
// Take following code from xlsx@0.15.1.
// They are private scoped and inaccessible from outside of the library.
//
const basedate = new Date(1899, 11, 30, 0, 0, 0);
const dnthresh = basedate.getTime() + (new Date().getTimezoneOffset() - basedate.getTimezoneOffset()) * 60000;

const day_ms = 24 * 60 * 60 * 1000;
const days_1462_ms = 1462 * day_ms;

function datenum(v, date1904) {
  let epoch = v.getTime();
  if (date1904) {
    epoch -= days_1462_ms;
  }
  return (epoch - dnthresh) / day_ms;
}
// -------------------------------------------------

function fixImportedDate(date, isDate1904) {
  const parsed = xlsx.SSF.parse_date_code(datenum(date, false), {date1904: isDate1904});
  // return `${parsed.y}-${parsed.m}-${parsed.d}`;
  return new Date(parsed.y, parsed.m, parsed.d, parsed.H, parsed.M, parsed.S);
}

function useSSFOutput() {
  const wb = xlsx.readFile('./tz_test_dates.xlsx', {cellDates: true});
  const sheet = wb.Sheets[(wb.SheetNames[0])];
  // original output
  const converted = xlsx.utils.sheet_to_json(sheet, {header: 1, cellDates:true});
  // apply hotfix
  const isDate1904 = wb.Workbook.WBProps.date1904;
  const fixed = converted.map((arr) => arr.map((v) => {
    if (v instanceof Date) {
      return fixImportedDate(v, isDate1904);
    } else {
      return v;
    }
  }));
  console.log('useSSFOutput:');
  console.log(fixed.map(arr => arr.map(v => v.toString())));
}

useSSFOutput();

Above code gives following result:

On LosAngeles timezone:

showPrevisionLossHotfix:
[ [ 'Mon Dec 31 2018 23:00:00 GMT-0800 (Pacific Standard Time)',
    'Thu Dec 31 1959 23:00:00 GMT-0800 (Pacific Standard Time)',
    'Tue Dec 31 1907 23:00:00 GMT-0800 (Pacific Standard Time)' ],
  [ 'Mon Dec 31 2018 23:00:00 GMT-0800 (Pacific Standard Time)',
    'Thu Feb 28 2019 23:00:00 GMT-0800 (Pacific Standard Time)',
    'Wed May 01 2019 00:00:00 GMT-0700 (Pacific Daylight Time)',
    'Mon Jul 01 2019 00:00:00 GMT-0700 (Pacific Daylight Time)',
    'Sun Sep 01 2019 00:00:00 GMT-0700 (Pacific Daylight Time)',
    'Fri Nov 01 2019 00:00:00 GMT-0700 (Pacific Daylight Time)' ] ]
useSSFOutput:
[ [ 'Fri Feb 01 2019 00:00:00 GMT-0800 (Pacific Standard Time)',
    'Mon Feb 01 1960 00:00:00 GMT-0800 (Pacific Standard Time)',
    'Sat Feb 01 1908 00:00:00 GMT-0800 (Pacific Standard Time)' ],
  [ 'Fri Feb 01 2019 00:00:00 GMT-0800 (Pacific Standard Time)',
    'Mon Apr 01 2019 00:00:00 GMT-0700 (Pacific Daylight Time)',
    'Sat Jun 01 2019 00:00:00 GMT-0700 (Pacific Daylight Time)',
    'Thu Aug 01 2019 00:00:00 GMT-0700 (Pacific Daylight Time)',
    'Tue Oct 01 2019 00:00:00 GMT-0700 (Pacific Daylight Time)',
    'Sun Dec 01 2019 00:00:00 GMT-0800 (Pacific Standard Time)' ] ]

On Asia/Seoul timezone:

showPrevisionLossHotfix:
[ [ 'Tue Jan 01 2019 00:00:00 GMT+0900 (Korean Standard Time)',
    'Thu Dec 31 1959 23:30:00 GMT+0830 (Korean Standard Time)',
    'Tue Dec 31 1907 23:27:52 GMT+0827 (Korean Standard Time)' ],
  [ 'Tue Jan 01 2019 00:00:00 GMT+0900 (Korean Standard Time)',
    'Fri Mar 01 2019 00:00:00 GMT+0900 (Korean Standard Time)',
    'Wed May 01 2019 00:00:00 GMT+0900 (Korean Standard Time)',
    'Mon Jul 01 2019 00:00:00 GMT+0900 (Korean Standard Time)',
    'Sun Sep 01 2019 00:00:00 GMT+0900 (Korean Standard Time)',
    'Fri Nov 01 2019 00:00:00 GMT+0900 (Korean Standard Time)' ] ]
useSSFOutput:
[ [ 'Fri Feb 01 2019 00:00:00 GMT+0900 (Korean Standard Time)',
    'Mon Feb 01 1960 00:00:00 GMT+0830 (Korean Standard Time)',
    'Sat Feb 01 1908 00:00:00 GMT+0827 (Korean Standard Time)' ],
  [ 'Fri Feb 01 2019 00:00:00 GMT+0900 (Korean Standard Time)',
    'Mon Apr 01 2019 00:00:00 GMT+0900 (Korean Standard Time)',
    'Sat Jun 01 2019 00:00:00 GMT+0900 (Korean Standard Time)',
    'Thu Aug 01 2019 00:00:00 GMT+0900 (Korean Standard Time)',
    'Tue Oct 01 2019 00:00:00 GMT+0900 (Korean Standard Time)',
    'Sun Dec 01 2019 00:00:00 GMT+0900 (Korean Standard Time)' ] ]