From 81c6608c6507409581659fff8c83ae20e44f5101 Mon Sep 17 00:00:00 2001 From: Xantorohara Date: Mon, 25 Jan 2021 10:53:46 +0300 Subject: [PATCH] Feature/80 date formats (#84) * #80 support output of dates as Java Date, LocalDate or LocalDateTime, Epoch seconds or SAS value --- .../com/epam/parso/date/OutputDateType.java | 29 ++ .../com/epam/parso/date/SasDateFormat.java | 245 +++++++++++++++ .../epam/parso/date/SasDateTimeFormat.java | 92 ++++++ .../epam/parso/date/SasTemporalFormatter.java | 281 ++++++++++++++++++ .../com/epam/parso/date/SasTimeFormat.java | 41 +++ .../com/epam/parso/impl/SasDateFormat.java | 101 ------- .../com/epam/parso/impl/SasFileConstants.java | 16 + .../com/epam/parso/impl/SasFileParser.java | 110 +++++-- .../epam/parso/impl/SasFileReaderImpl.java | 16 + src/test/java/com/epam/parso/BugsTest.java | 30 -- .../java/com/epam/parso/SasDateTypeTest.java | 209 +++++++++++++ src/test/resources/bugs/81-dates.sas7bdat | Bin 131072 -> 0 bytes src/test/resources/csv/dates_leap_days.csv | 15 - .../resources/csv/dates_leap_days_meta.csv | 23 -- .../dates/sas7bdat/date_format_date.sas7bdat | Bin 0 -> 8192 bytes .../sas7bdat/date_format_datetime.sas7bdat | Bin 0 -> 65536 bytes .../dates/sas7bdat/date_format_time.sas7bdat | Bin 0 -> 8192 bytes .../sas7bdat/date_leap_days.sas7bdat} | Bin 8192 -> 8192 bytes .../resources/sas7bdat/dates_leap_days.sas | 57 ---- 19 files changed, 1013 insertions(+), 252 deletions(-) create mode 100644 src/main/java/com/epam/parso/date/OutputDateType.java create mode 100644 src/main/java/com/epam/parso/date/SasDateFormat.java create mode 100644 src/main/java/com/epam/parso/date/SasDateTimeFormat.java create mode 100644 src/main/java/com/epam/parso/date/SasTemporalFormatter.java create mode 100644 src/main/java/com/epam/parso/date/SasTimeFormat.java delete mode 100644 src/main/java/com/epam/parso/impl/SasDateFormat.java create mode 100644 src/test/java/com/epam/parso/SasDateTypeTest.java delete mode 100644 src/test/resources/bugs/81-dates.sas7bdat delete mode 100644 src/test/resources/csv/dates_leap_days.csv delete mode 100644 src/test/resources/csv/dates_leap_days_meta.csv create mode 100644 src/test/resources/dates/sas7bdat/date_format_date.sas7bdat create mode 100644 src/test/resources/dates/sas7bdat/date_format_datetime.sas7bdat create mode 100644 src/test/resources/dates/sas7bdat/date_format_time.sas7bdat rename src/test/resources/{sas7bdat/dates_leap_days.sas7bdat => dates/sas7bdat/date_leap_days.sas7bdat} (82%) delete mode 100644 src/test/resources/sas7bdat/dates_leap_days.sas diff --git a/src/main/java/com/epam/parso/date/OutputDateType.java b/src/main/java/com/epam/parso/date/OutputDateType.java new file mode 100644 index 0000000..def7ccc --- /dev/null +++ b/src/main/java/com/epam/parso/date/OutputDateType.java @@ -0,0 +1,29 @@ +package com.epam.parso.date; + +/** + * Option how Parso outputs dates. + */ +public enum OutputDateType { + /** + * Outputs date as java.util.Date for date and datetime formats. + * Note that time will be represented as a number. + */ + JAVA_DATE_LEGACY, + /** + * Outputs date as LocalDate and LocalDateTime for date and datetime formats. + * Note that time will be represented as a number, because of SAS time + * can't be represented as a LocalTime! + */ + JAVA_TEMPORAL, + /** + * Output date as raw SAS value. + * It is number of days (for date type) or seconds(for datetime type) + * since 1 January 1960 as java double. + */ + SAS_VALUE, + /** + * Output date as number of seconds since 1 January 1970 as java double. + * Note that it may contain fractional part. + */ + EPOCH_SECONDS +} diff --git a/src/main/java/com/epam/parso/date/SasDateFormat.java b/src/main/java/com/epam/parso/date/SasDateFormat.java new file mode 100644 index 0000000..6b96329 --- /dev/null +++ b/src/main/java/com/epam/parso/date/SasDateFormat.java @@ -0,0 +1,245 @@ +package com.epam.parso.date; + +/** + * Collection of SAS date formats. + */ +enum SasDateFormat { + /** + * Writes date values in the form ddmmmyy or ddmmmyyyy. + * See: https://v8doc.sas.com/sashtml/lgref/z0195834.htm + */ + DATE, + /** + * Writes date values as the day of the month. + * See: https://v8doc.sas.com/sashtml/lgref/z0201472.htm + */ + DAY, + /** + * Writes date values in the form ddmmyy or ddmmyyyy. + * https://v8doc.sas.com/sashtml/lgref/z0197953.htm + * See also: + * https://v8doc.sas.com/sashtml/lgref/z0590669.htm + */ + DDMMYY, + /** + * DDMMYYB with a blank separator. + */ + DDMMYYB, + /** + * DDMMYYC with a colon separator. + */ + DDMMYYC, + /** + * DDMMYYD with a dash separator. + */ + DDMMYYD, + /** + * DDMMYY with N indicates no separator. + * When x is N, the width range is 2-8. + */ + DDMMYYN, + /** + * DDMMYYP with a period separator. + */ + DDMMYYP, + /** + * DDMMYYS with a slash separator. + */ + DDMMYYS, + /** + * Writes date values in the form mmddyy or mmddyyyy. + * https://v8doc.sas.com/sashtml/lgref/z0199367.htm + * See also: + * https://v8doc.sas.com/sashtml/lgref/z0590662.htm + */ + MMDDYY, + /** + * MMDDYYB with a blank separator. + */ + MMDDYYB, + /** + * MMDDYYC with a colon separator. + */ + MMDDYYC, + /** + * MMDDYYD with a dash separator.. + */ + MMDDYYD, + /** + * MMDDYY with N indicates no separator. + * When x is N, the width range is 2-8. + */ + MMDDYYN, + /** + * MMDDYYP with a period separator. + */ + MMDDYYP, + /** + * MMDDYYS with a slash separator. + */ + MMDDYYS, + /** + * Writes date values in the form yymmdd or yyyymmdd. + * https://v8doc.sas.com/sashtml/lgref/z0197961.htm + * See also: + * https://v8doc.sas.com/sashtml/lgref/z0589916.htm + */ + YYMMDD, + /** + * YYMMDDB with a blank separator. + */ + YYMMDDB, + /** + * YYMMDDC with a colon separator. + */ + YYMMDDC, + /** + * YYMMDDD with a dash separator. + */ + YYMMDDD, + /** + * YYMMDD with N indicates no separator. + * When x is N, the width range is 2-8. + */ + YYMMDDN, + /** + * YYMMDDP with a period separator. + */ + YYMMDDP, + /** + * YYMMDDS with a slash separator. + */ + YYMMDDS, + /** + * Writes date values as the month and the year and separates them with a character. + * https://v8doc.sas.com/sashtml/lgref/z0199314.htm + * MMYY with a M separator. + */ + MMYY, + /** + * MMYYC with a colon separator. + */ + MMYYC, + /** + * MMYYD with a dash separator.. + */ + MMYYD, + /** + * MMYY with N indicates no separator. + * When no separator is specified, the width range is 4-32 and the default changes to 6. + */ + MMYYN, + /** + * MMYYP with a period separator. + */ + MMYYP, + /** + * MMYYS with a slash separator. + */ + MMYYS, + /** + * Writes date values as the year and month and separates them with a character. + * https://v8doc.sas.com/sashtml/lgref/z0199309.htm + * YYMM with a M separator. + */ + YYMM, + /** + * YYMMC with a colon separator. + */ + YYMMC, + /** + * YYMMD with a dash separator.. + */ + YYMMD, + /** + * YYMM with N indicates no separator. + * When no separator is specified, the width range is 4-32 and the default changes to 6. + */ + YYMMN, + /** + * YYMMP with a period separator. + */ + YYMMP, + /** + * YYMMS with a slash separator. + */ + YYMMS, + /** + * Writes date values as Julian dates in the form yyddd or yyyyddd. + * See: https://v8doc.sas.com/sashtml/lgref/z0197940.htm + */ + JULIAN, + /** + * Writes date values as the Julian day of the year. + * See: https://v8doc.sas.com/sashtml/lgref/z0205162.htm + */ + JULDAY, + /** + * Writes date values as the month. + * See: https://v8doc.sas.com/sashtml/lgref/z0171689.htm + * Note that MONTH1. returns HEX value. + */ + MONTH, + /** + * Writes date values as the year. + * See: https://v8doc.sas.com/sashtml/lgref/z0205234.htm + */ + YEAR, + /** + * Writes date values as the month and the year in the form mmmyy or mmmyyyy. + * See: https://v8doc.sas.com/sashtml/lgref/z0197959.htm + */ + MONYY, + /** + * Writes date values as the year and the month abbreviation. + * See: https://v8doc.sas.com/sashtml/lgref/z0205240.htm + */ + YYMON, + /** + * Writes date values by using the ISO 8601 basic notation yyyymmdd. + */ + B8601DA, + /** + * Writes date values by using the ISO 8601 extended notation yyyy-mm-dd. + */ + E8601DA, + /** + * Writes date values as the name of the month. + * See: https://v8doc.sas.com/sashtml/lgref/z0201049.htm + */ + MONNAME, + /** + * Writes date values as the day of the week and the date in the form day-of-week, + * month-name dd, yy (or yyyy). + * See: https://v8doc.sas.com/sashtml/lgref/z0201433.htm + */ + WEEKDATE, + /** + * Writes date values as day of week and date in the form day-of-week, + * dd month-name yy (or yyyy). + * See: https://v8doc.sas.com/sashtml/lgref/z0201303.htm + */ + WEEKDATX, + /** + * Writes date values as the day of the week. + * See: https://v8doc.sas.com/sashtml/lgref/z0200757.htm + */ + WEEKDAY, + /** + * Writes date values as the name of the day of the week. + * See: https://v8doc.sas.com/sashtml/lgref/z0200842.htm + */ + DOWNAME, + /** + * Writes date values as the name of the month, + * the day, and the year in the form month-name dd, yyyy. + * See: https://v8doc.sas.com/sashtml/lgref/z0201451.htm + */ + WORDDATE, + /** + * Writes date values as the day, the name of the month, + * and the year in the form dd month-name yyyy. + * See: https://v8doc.sas.com/sashtml/lgref/z0201147.htm + */ + WORDDATX +} diff --git a/src/main/java/com/epam/parso/date/SasDateTimeFormat.java b/src/main/java/com/epam/parso/date/SasDateTimeFormat.java new file mode 100644 index 0000000..96bd099 --- /dev/null +++ b/src/main/java/com/epam/parso/date/SasDateTimeFormat.java @@ -0,0 +1,92 @@ +package com.epam.parso.date; + +/** + * Collection of SAS datetime formats. + */ +enum SasDateTimeFormat { + /** + * Writes datetime values in the form ddmmmyy:hh:mm:ss.ss. + * See: https://v8doc.sas.com/sashtml/lgref/z0197923.htm + */ + DATETIME, + /** + * Writes dates from datetime values by using the ISO 8601 basic notation yyyymmdd. + */ + B8601DN, + /** + * Writes datetime values by using the ISO 8601 basic notation yyyymmddThhmmss. + */ + B8601DT, + /** + * Adjusts a Coordinated Universal Time (UTC) datetime value to the user local date and time. + * Then, writes the local date and time by using the ISO 8601 datetime + * and time zone basic notation yyyymmddThhmmss+hhmm. + */ + B8601DX, + /** + * Reads Coordinated Universal Time (UTC) datetime values that are specified using the + * ISO 8601 datetime basic notation yyyymmddThhmmss+|–hhmm or yyyymmddThhmmssZ. + */ + B8601DZ, + /** + * Writes datetime values as local time by appending a time zone offset difference between the local time and UTC, + * using the ISO 8601 basic notation yyyymmddThhmmss+|–hhmm. + */ + B8601LX, + /** + * Writes dates from SAS datetime values by using the ISO 8601 extended notation yyyy-mm-dd. + */ + E8601DN, + /** + * Reads datetime values that are specified using the + * ISO 8601 extended notation yyyy-mm-ddThh:mm:ss.. + */ + E8601DT, + /** + * Adjusts a Coordinated Universal Time (UTC) datetime value to the user local date and time. + * Then, writes the local date and time by using the ISO 8601 datetime + * and time zone extended notation yyyy-mm-ddThh:mm:ss+hh:mm. + */ + E8601DX, + /** + * Reads Coordinated Universal Time (UTC) datetime values that are specified using the ISO 8601 + * datetime extended notation yyyy-mm-ddThh:mm:ss+|–hh:mm. or yyyy-mm-ddThh:mm:ss.Z. + */ + E8601DZ, + /** + * Writes datetime values as local time by appending a time zone offset difference between the local time and UTC, + * using the ISO 8601 extended notation yyyy-mm-ddThh:mm:ss+|–hh:mm. + */ + E8601LX, + /** + * Writes datetime values in the form ddmmmyy:hh:mm:ss.ss with AM or PM. + * See: https://v8doc.sas.com/sashtml/lgref/z0196050.htm + */ + DATEAMPM, + /** + * Expects a datetime value as input and writes date values in the form ddmmmyy or ddmmmyyyy. + */ + DTDATE, + /** + * Writes the date part of a datetime value as the month and year in the form mmmyy or mmmyyyy. + */ + DTMONYY, + /** + * Writes the date part of a SAS datetime value as the day of the week and the date in the form + * day-of-week, dd month-name yy (or yyyy). + */ + DTWKDATX, + /** + * Writes the date part of a SAS datetime value as the year in the form yy or yyyy. + */ + DTYEAR, + /** + * Writes datetime values in the form mm/dd/yy hh:mm AM|PM. The year can be either two or four digits. + */ + MDYAMPM, + /** + * Writes the time portion of datetime values in the form hh:mm:ss.ss. + * See: https://v8doc.sas.com/sashtml/lgref/z0201157.htm + */ + TOD +} diff --git a/src/main/java/com/epam/parso/date/SasTemporalFormatter.java b/src/main/java/com/epam/parso/date/SasTemporalFormatter.java new file mode 100644 index 0000000..86e2c80 --- /dev/null +++ b/src/main/java/com/epam/parso/date/SasTemporalFormatter.java @@ -0,0 +1,281 @@ +package com.epam.parso.date; + +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.util.Date; + +import static com.epam.parso.date.OutputDateType.SAS_VALUE; + + +/** + * SAS supports wide family of date formats. + * This class allows to represent SAS date in various types. + *

+ * This class is not thread-safe and it should be synchronised externally. + * Actually, it is not a problem for the the Parso itself as it is a single-threaded + * library, where each instance of SasFileParser has it's own instance of the formatter. + */ +public class SasTemporalFormatter { + + /** + * Seconds in day: 86400. + */ + private static final int SECONDS_IN_DAY = 60 * 60 * 24; + + /** + * The difference in days between 01/01/1960 (the dates starting point in SAS) + * and 01/01/1970 (the dates starting point in Java). + */ + private static final int SAS_VS_EPOCH_DIFF_DAYS = 365 * 10 + 3; + + /** + * The difference in seconds between 01/01/1960 (the dates starting point in SAS) + * and 01/01/1970 (the dates starting point in Java). + */ + private static final int SAS_VS_EPOCH_DIFF_SECONDS = SECONDS_IN_DAY * SAS_VS_EPOCH_DIFF_DAYS; + + /** + * Milliseconds in one second. + */ + private static final int MILLIS_IN_SECOND = 1000; + + /** + * First time when a leap day is removed from the SAS calendar. + * In seconds since 1960-01-01 + */ + private static final double SAS_SECONDS_29FEB4000 = 64381305600D; + + /** + * Second time when a leap day is removed from the SAS calendar. + * In seconds since 1960-01-01 + */ + private static final double SAS_SECONDS_29FEB8000 = 190609027200D; + + /** + * SAS removes leap day every 4000 year. + * It removes these days: + * - 29FEB4000 + * - 29FEB8000 + * This guy proposed such approach many years ago: https://en.wikipedia.org/wiki/John_Herschel + *

+ * Sometimes people discussed why SAS dates are so strange: + * - https://blogs.sas.com/content/sasdummy/2010/04/05/in-the-year-9999/ + * - https://communities.sas.com/t5/SAS-Programming/Leap-Years-divisible-by-4000/td-p/663467 + *

+ * See the SAS program and its output: + * ```shell + * data test; + * dtime = '28FEB4000:00:00:00'dt; + * put dtime; *out: 64381219200 + *

+ * dtime = '29FEB4000:00:00:00'dt; + * put dtime; *err: ERROR: Invalid date/time/datetime constant '29FEB4000:00:00:00'dt. + *

+ * dtime = '01MAR4000:00:00:00'dt; + * put dtime; *out: 64381305600 + *

+ * dtime = '31DEC4000:00:00:00'dt; + * put dtime; *out: 64407657600 + *

+ * dtime = '28FEB8000:00:00:00'dt; + * put dtime; *out: 190608940800 + *

+ * dtime = '29FEB8000:00:00:00'dt; + * put dtime; *err: ERROR: Invalid date/time/datetime constant '29FEB8000:00:00:00'dt. + *

+ * dtime = '01MAR8000:00:00:00'dt; + * put dtime; * out: 190609027200 + *

+ * dtime = '31DEC8000:00:00:00'dt; + * put dtime; *out: 190635379200 + *

+ * dtime = '31DEC9999:00:00:00'dt; + * put dtime; *out: 253717660800 + * run; + * ``` + * As you can see SAS doesn't accept leap days for 4000 and 8000 years + * and removes these days at all from the SAS calendar. + *

+ * At the same time these leap days are ok for: + * - Java: `LocalDateTime.of(4000, 2, 29, 0, 0).toEpochSecond(ZoneOffset.UTC)` + * outputs 64065686400 + * - JavaScript: `Date.parse('4000-02-29')` + * outputs 64065686400000 + * - GNU/date: `date --utc --date '4000-02-29' +%s` + * outputs 64065686400 + * and so on. + *

+ * So, in order to parse SAS dates correctly, + * we need to restore removed leap days + * + * @param sasSeconds SAS date representation in seconds since 1960-01-01 + * @return seconds with restored leap days + */ + private static double sasLeapDaysFix(double sasSeconds) { + if (sasSeconds >= SAS_SECONDS_29FEB4000) { + if (sasSeconds >= SAS_SECONDS_29FEB8000) { + sasSeconds += SECONDS_IN_DAY; //restore Y8K leap day + } + sasSeconds += SECONDS_IN_DAY; //restore Y4K leap day + } + return sasSeconds; + } + + /** + * Format SAS seconds explicitly into the java Date. + * + * @param sasSeconds seconds since 1960-01-01 + * @return date + */ + public Date formatSasSecondsAsJavaDate(double sasSeconds) { + double epochSeconds = sasLeapDaysFix(sasSeconds) - SAS_VS_EPOCH_DIFF_SECONDS; + return new Date((long) (epochSeconds * MILLIS_IN_SECOND)); + } + + /** + * Format SAS date in SAS days to one of the specified form. + * + * @param sasDays days since 1960-01-01 + * @param dateFormatType type of output date + * @param sasFormatName date column format name + * @param width date column format width + * @param precision date column format precision + * @return date representation + */ + public Object formatSasDate(Double sasDays, OutputDateType dateFormatType, + String sasFormatName, int width, int precision) { + + if (dateFormatType == SAS_VALUE) { + return sasDays; + } else if (sasDays == null || Double.isNaN(sasDays)) { + return null; + } + + double epochSeconds = sasLeapDaysFix(sasDays * SECONDS_IN_DAY) - SAS_VS_EPOCH_DIFF_SECONDS; + + switch (dateFormatType) { + case EPOCH_SECONDS: + return epochSeconds; + case JAVA_TEMPORAL: + return LocalDate.ofEpochDay((long) (epochSeconds / SECONDS_IN_DAY)); + case JAVA_DATE_LEGACY: + default: + return new Date((long) (epochSeconds * MILLIS_IN_SECOND)); + } + } + + /** + * Format SAS time in SAS seconds to one of the specified form. + * For the compatibility with Parso this formatter returns number + * (long or double) instead of date case of JAVA_DATE output type. + * + * @param sasSeconds days since 1960-01-01 + * @param dateFormatType type of output date + * @param sasFormatName date column format name + * @param width date column format width + * @param precision date column format precision + * @return date representation + */ + public Object formatSasTime(Double sasSeconds, OutputDateType dateFormatType, + String sasFormatName, int width, int precision) { + if (dateFormatType == SAS_VALUE) { + return sasSeconds; + } else if (sasSeconds == null || Double.isNaN(sasSeconds)) { + return null; + } + + switch (dateFormatType) { + case JAVA_DATE_LEGACY: + case JAVA_TEMPORAL: + default: + // These lines below for compatibility with existing Parso result. + // Number of seconds in Parso is represented in some cases as long + // or as double using the SasFileParser.convertByteArrayToNumber function. + long longSeconds = Math.round(sasSeconds); + if (Math.abs(sasSeconds - longSeconds) > 0) { + return sasSeconds; + } else { + return longSeconds; + } + } + } + + /** + * Format SAS date-time in SAS seconds to one of the specified form. + * + * @param sasSeconds seconds since midnight + * @param dateFormatType type of output date + * @param sasFormatName date column format name + * @param width date column format width + * @param precision date column format precision + * @return date representation + */ + public Object formatSasDateTime(Double sasSeconds, OutputDateType dateFormatType, + String sasFormatName, int width, int precision) { + if (dateFormatType == SAS_VALUE) { + return sasSeconds; + } else if (sasSeconds == null || Double.isNaN(sasSeconds)) { + return null; + } + + double epochSeconds = sasLeapDaysFix(sasSeconds) - SAS_VS_EPOCH_DIFF_SECONDS; + + switch (dateFormatType) { + case EPOCH_SECONDS: + return epochSeconds; + case JAVA_TEMPORAL: + String f = String.format("%.9f", epochSeconds); + int nanos = Integer.parseInt(f.substring(f.indexOf('.') + 1)); + return LocalDateTime.ofEpochSecond((long) epochSeconds, nanos, ZoneOffset.UTC); + case JAVA_DATE_LEGACY: + default: + return new Date((long) (epochSeconds * MILLIS_IN_SECOND)); + } + } + + /** + * Check if the specified SAS format is type of date. + * + * @param sasFormatName SAS format name + * @return true if matched + */ + public static boolean isDateFormat(String sasFormatName) { + for (SasDateFormat s : SasDateFormat.values()) { + if (s.name().equals(sasFormatName)) { + return true; + } + } + return false; + } + + /** + * Check if the specified SAS format is type of time. + * + * @param sasFormatName SAS format name + * @return true if matched + */ + public static boolean isTimeFormat(String sasFormatName) { + for (SasTimeFormat s : SasTimeFormat.values()) { + if (s.name().equals(sasFormatName)) { + return true; + } + } + return false; + } + + /** + * Check if the specified SAS format is type of date-time. + * + * @param sasFormatName SAS format name + * @return true if matched + */ + public static boolean isDateTimeFormat(String sasFormatName) { + for (SasDateTimeFormat s : SasDateTimeFormat.values()) { + if (s.name().equals(sasFormatName)) { + return true; + } + } + return false; + } +} diff --git a/src/main/java/com/epam/parso/date/SasTimeFormat.java b/src/main/java/com/epam/parso/date/SasTimeFormat.java new file mode 100644 index 0000000..ee236c0 --- /dev/null +++ b/src/main/java/com/epam/parso/date/SasTimeFormat.java @@ -0,0 +1,41 @@ +package com.epam.parso.date; + +/** + * Collection of SAS time formats. + */ +public enum SasTimeFormat { + /** + * Writes time values as hours, minutes, and seconds in the form hh:mm:ss.ss. + * See: https://v8doc.sas.com/sashtml/lgref/z0197928.htm + */ + TIME, + /** + * Writes time values as the number of minutes and seconds since midnight. + * See: https://v8doc.sas.com/sashtml/lgref/z0198053.htm + */ + MMSS, + /** + * Writes time values as hours and minutes in the form hh:mm. + * See: https://v8doc.sas.com/sashtml/lgref/z0198049.htm + */ + HHMM, + /** + * Writes time values as hours and decimal fractions of hours. + * See: https://v8doc.sas.com/sashtml/lgref/z0198051.htm + */ + HOUR, + /** + * Writes time values as hours, minutes, and seconds in the form hh:mm:ss.ss with AM or PM. + * See: https://v8doc.sas.com/sashtml/lgref/z0201272.htm + */ + TIMEAMPM, + /** + * Writes time values as local time, appending the Coordinated Universal Time (UTC) offset + * for the local SAS session, using the ISO 8601 extended time notation hh:mm:ss+|–hh:mm. + */ + E8601LZ, + /** + * Writes time values by using the ISO 8601 extended notation hh:mm:ss.ffffff. + */ + E8601TM +} diff --git a/src/main/java/com/epam/parso/impl/SasDateFormat.java b/src/main/java/com/epam/parso/impl/SasDateFormat.java deleted file mode 100644 index 27577c2..0000000 --- a/src/main/java/com/epam/parso/impl/SasDateFormat.java +++ /dev/null @@ -1,101 +0,0 @@ -package com.epam.parso.impl; - -import static com.epam.parso.impl.SasFileConstants.SECONDS_IN_DAY; - -/** - * SAS supports wide family of date formats. - * It is reasonable to keep all SAS date related features separately. - * See more about SAS dates: - * - https://v8doc.sas.com/sashtml/lrcon/zenid-63.htm - * - https://v8doc.sas.com/sashtml/lgref/z0197923.htm - * - https://v8doc.sas.com/sashtml/ets/chap2/sect7.htm - */ -final class SasDateFormat { - /** - * Private constructor for utility class. - */ - private SasDateFormat() { - } - - /** - * First time when a leap day is removed from the SAS calendar. - * In seconds since 1960-01-01 - */ - private static final double SAS_SECONDS_29FEB4000 = 64381305600D; - - /** - * Second time when a leap day is removed from the SAS calendar. - * In seconds since 1960-01-01 - */ - private static final double SAS_SECONDS_29FEB8000 = 190609027200D; - - /** - * SAS removes leap day every 4000 year. - * It removes these days: - * - 29FEB4000 - * - 29FEB8000 - * This guy proposed such approach many years ago: https://en.wikipedia.org/wiki/John_Herschel - *

- * Sometimes people discussed why SAS dates are so strange: - * - https://blogs.sas.com/content/sasdummy/2010/04/05/in-the-year-9999/ - * - https://communities.sas.com/t5/SAS-Programming/Leap-Years-divisible-by-4000/td-p/663467 - *

- * See the SAS program and its output: - * ```shell - * data test; - * dtime = '28FEB4000:00:00:00'dt; - * put dtime; *out: 64381219200 - *

- * dtime = '29FEB4000:00:00:00'dt; - * put dtime; *err: ERROR: Invalid date/time/datetime constant '29FEB4000:00:00:00'dt. - *

- * dtime = '01MAR4000:00:00:00'dt; - * put dtime; *out: 64381305600 - *

- * dtime = '31DEC4000:00:00:00'dt; - * put dtime; *out: 64407657600 - *

- * dtime = '28FEB8000:00:00:00'dt; - * put dtime; *out: 190608940800 - *

- * dtime = '29FEB8000:00:00:00'dt; - * put dtime; *err: ERROR: Invalid date/time/datetime constant '29FEB8000:00:00:00'dt. - *

- * dtime = '01MAR8000:00:00:00'dt; - * put dtime; * out: 190609027200 - *

- * dtime = '31DEC8000:00:00:00'dt; - * put dtime; *out: 190635379200 - *

- * dtime = '31DEC9999:00:00:00'dt; - * put dtime; *out: 253717660800 - * run; - * ``` - * As you can see SAS doesn't accept leap days for 4000 and 8000 years - * and removes these days at all from the SAS calendar. - *

- * At the same time these leap days are ok for: - * - Java: `LocalDateTime.of(4000, 2, 29, 0, 0).toEpochSecond(ZoneOffset.UTC)` - * outputs 64065686400 - * - JavaScript: `Date.parse('4000-02-29')` - * outputs 64065686400000 - * - GNU/date: `date --utc --date '4000-02-29' +%s` - * outputs 64065686400 - * and so on. - *

- * So, in order to parse SAS dates correctly, - * we need to restore removed leap days - * - * @param sasSeconds SAS date representation in seconds since 1960-01-01 - * @return seconds with restored leap days - */ - public static double sasLeapDaysFix(double sasSeconds) { - if (sasSeconds >= SAS_SECONDS_29FEB4000) { - if (sasSeconds >= SAS_SECONDS_29FEB8000) { - sasSeconds += SECONDS_IN_DAY; //restore Y8K leap day - } - sasSeconds += SECONDS_IN_DAY; //restore Y4K leap day - } - return sasSeconds; - } -} diff --git a/src/main/java/com/epam/parso/impl/SasFileConstants.java b/src/main/java/com/epam/parso/impl/SasFileConstants.java index 7f328e0..8ee12a4 100644 --- a/src/main/java/com/epam/parso/impl/SasFileConstants.java +++ b/src/main/java/com/epam/parso/impl/SasFileConstants.java @@ -1048,44 +1048,60 @@ public interface SasFileConstants { /** * The number of milliseconds in a second. + * Deprecated: dates-related functionality moved to date package. */ + @Deprecated long MILLISECONDS_IN_SECONDS = 1000L; /** * The number of seconds in a minute. + * Deprecated: dates-related functionality moved to date package. */ + @Deprecated int SECONDS_IN_MINUTE = 60; /** * The number of minutes in an hour. + * Deprecated: dates-related functionality moved to date package. */ + @Deprecated int MINUTES_IN_HOUR = 60; /** * The number of hours in a day. + * Deprecated: dates-related functionality moved to date package. */ + @Deprecated int HOURS_IN_DAY = 24; /** * The number of days in a non-leap year. + * Deprecated: dates-related functionality moved to date package. */ + @Deprecated int DAYS_IN_YEAR = 365; /** * The difference in days between 01/01/1960 (the dates starting point in SAS) and 01/01/1970 (the dates starting * point in Java). + * Deprecated: dates-related functionality moved to date package. */ + @Deprecated int START_DATES_DAYS_DIFFERENCE = DAYS_IN_YEAR * 10 + 3; /** * The number of seconds in a day. + * Deprecated: dates-related functionality moved to date package. */ + @Deprecated int SECONDS_IN_DAY = SECONDS_IN_MINUTE * MINUTES_IN_HOUR * HOURS_IN_DAY; /** * The difference in seconds between 01/01/1960 (the dates starting point in SAS) and 01/01/1970 (the dates starting * point in Java). + * Deprecated: no reason to make it public. */ + @Deprecated int START_DATES_SECONDS_DIFFERENCE = SECONDS_IN_DAY * START_DATES_DAYS_DIFFERENCE; /** diff --git a/src/main/java/com/epam/parso/impl/SasFileParser.java b/src/main/java/com/epam/parso/impl/SasFileParser.java index 157dcf8..a344177 100644 --- a/src/main/java/com/epam/parso/impl/SasFileParser.java +++ b/src/main/java/com/epam/parso/impl/SasFileParser.java @@ -23,6 +23,8 @@ import com.epam.parso.ColumnFormat; import com.epam.parso.ColumnMissingInfo; import com.epam.parso.SasFileProperties; +import com.epam.parso.date.OutputDateType; +import com.epam.parso.date.SasTemporalFormatter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -35,8 +37,6 @@ import java.nio.ByteOrder; import java.util.*; -import static com.epam.parso.impl.DateTimeConstants.DATETIME_FORMAT_STRINGS; -import static com.epam.parso.impl.DateTimeConstants.DATE_FORMAT_STRINGS; import static com.epam.parso.impl.ParserMessageConstants.BLOCK_COUNT; import static com.epam.parso.impl.ParserMessageConstants.COLUMN_FORMAT; import static com.epam.parso.impl.ParserMessageConstants.EMPTY_INPUT_STREAM; @@ -122,6 +122,12 @@ public final class SasFileParser { * The flag of data output in binary or string format. */ private final Boolean byteOutput; + + /** + * Output date type. + */ + private final OutputDateType outputDateType; + /** * The list of current page data subheaders. */ @@ -244,6 +250,11 @@ public final class SasFileParser { */ private String deletedMarkers = ""; + /** + * Instance of SasDateFormatter. + */ + private final SasTemporalFormatter sasTemporalFormatter = new SasTemporalFormatter(); + /** * The constructor that reads metadata from the sas7bdat, parses it and puts the results in * {@link SasFileParser#sasFileProperties}. @@ -253,6 +264,7 @@ public final class SasFileParser { private SasFileParser(Builder builder) { sasFileStream = new DataInputStream(builder.sasFileStream); byteOutput = builder.byteOutput; + outputDateType = builder.outputDateType; Map tmpMap = new HashMap<>(); tmpMap.put(SubheaderIndexes.ROW_SIZE_SUBHEADER_INDEX, new RowSizeSubheader()); @@ -883,14 +895,16 @@ private Object processElement(byte[] source, int offset, int currentColumnIndex) if (columns.get(currentColumnIndex).getFormat().getName().isEmpty()) { return convertByteArrayToNumber(temp); } else { - if (DATETIME_FORMAT_STRINGS.containsKey(columns.get(currentColumnIndex).getFormat().getName())) { - return bytesToDateTime(temp); + ColumnFormat columnFormat = columns.get(currentColumnIndex).getFormat(); + String sasDateFormat = columnFormat.getName(); + if (SasTemporalFormatter.isDateTimeFormat(sasDateFormat)) { + return bytesToDateTime(temp, outputDateType, columnFormat); + } else if (SasTemporalFormatter.isDateFormat(sasDateFormat)) { + return bytesToDate(temp, outputDateType, columnFormat); + } else if (SasTemporalFormatter.isTimeFormat(sasDateFormat)) { + return bytesToTime(temp, outputDateType, columnFormat); } else { - if (DATE_FORMAT_STRINGS.containsKey(columns.get(currentColumnIndex).getFormat().getName())) { - return bytesToDate(temp); - } else { - return convertByteArrayToNumber(temp); - } + return convertByteArrayToNumber(temp); } } } @@ -1060,9 +1074,8 @@ private String bytesToString(byte[] bytes, int offset, int length) } /** - * The function to convert an array of bytes that stores the number of seconds elapsed from 01/01/1960 into - * a variable of the {@link Date} type. The {@link SasFileConstants#DATE_TIME_FORMAT_STRINGS} variable stores - * the formats of the columns that store such data. + * The function to convert an array of bytes that stores the number of seconds + * elapsed from 01/01/1960 into a variable of the {@link Date} type. * * @param bytes an array of bytes that stores the type. * @return a variable of the {@link Date} type. @@ -1072,27 +1085,54 @@ private Date bytesToDateTime(byte[] bytes) { if (Double.isNaN(doubleSeconds)) { return null; } else { - double seconds = SasDateFormat.sasLeapDaysFix(doubleSeconds) - START_DATES_SECONDS_DIFFERENCE; - return new Date((long) (seconds * MILLISECONDS_IN_SECONDS)); + return sasTemporalFormatter.formatSasSecondsAsJavaDate(doubleSeconds); } } /** - * The function to convert an array of bytes that stores the number of days elapsed from 01/01/1960 into a variable - * of the {@link Date} type. {@link SasFileConstants#DATE_FORMAT_STRINGS} stores the formats of columns that contain - * such data. + * The function to convert an array of bytes that stores the number of seconds + * elapsed from 01/01/1960 into the date represented according to outputDateType. * - * @param bytes the array of bytes that stores the number of days from 01/01/1960. - * @return a variable of the {@link Date} type. + * @param bytes an array of bytes that stores the type. + * @param outputDateType type of the date formatting + * @param columnFormat SAS date format + * @return datetime representation + */ + private Object bytesToDateTime(byte[] bytes, OutputDateType outputDateType, ColumnFormat columnFormat) { + double doubleSeconds = bytesToDouble(bytes); + return sasTemporalFormatter.formatSasDateTime(doubleSeconds, outputDateType, + columnFormat.getName(), columnFormat.getWidth(), columnFormat.getPrecision()); + } + + /** + * The function to convert an array of bytes that stores the number of seconds + * since midnight into the date represented according to outputDateType. + * + * @param bytes an array of bytes that stores the type. + * @param outputDateType type of the date formatting + * @param columnFormat SAS date format + * @return time representation + */ + private Object bytesToTime(byte[] bytes, OutputDateType outputDateType, ColumnFormat columnFormat) { + double doubleSeconds = bytesToDouble(bytes); + return sasTemporalFormatter.formatSasTime(doubleSeconds, outputDateType, + columnFormat.getName(), columnFormat.getWidth(), columnFormat.getPrecision()); + } + + + /** + * The function to convert an array of bytes that stores the number of days + * elapsed from 01/01/1960 into the date represented according to outputDateType. + * + * @param bytes the array of bytes that stores the number of days from 01/01/1960. + * @param outputDateType type of the date formatting + * @param columnFormat SAS date format + * @return date representation */ - private Date bytesToDate(byte[] bytes) { + private Object bytesToDate(byte[] bytes, OutputDateType outputDateType, ColumnFormat columnFormat) { double doubleDays = bytesToDouble(bytes); - if (Double.isNaN(doubleDays)) { - return null; - } else { - double seconds = SasDateFormat.sasLeapDaysFix(doubleDays * SECONDS_IN_DAY) - START_DATES_SECONDS_DIFFERENCE; - return new Date((long) (seconds * MILLISECONDS_IN_SECONDS)); - } + return sasTemporalFormatter.formatSasDate(doubleDays, outputDateType, + columnFormat.getName(), columnFormat.getWidth(), columnFormat.getPrecision()); } /** @@ -1264,6 +1304,11 @@ private Builder() { */ private String encoding; + /** + * Default value for {@link SasFileParser#outputDateType} variable. + */ + private OutputDateType outputDateType = OutputDateType.JAVA_DATE_LEGACY; + /** * Default value for {@link SasFileParser#byteOutput} variable. */ @@ -1289,6 +1334,19 @@ public Builder encoding(String val) { return this; } + /** + * Sets the specified type of the output date format. + * + * @param val value to be set. + * @return result builder. + */ + public Builder outputDateType(OutputDateType val) { + if (val != null) { + outputDateType = val; + } + return this; + } + /** * The function to specify builders byteOutput variable. * diff --git a/src/main/java/com/epam/parso/impl/SasFileReaderImpl.java b/src/main/java/com/epam/parso/impl/SasFileReaderImpl.java index 117b62a..77dd81a 100644 --- a/src/main/java/com/epam/parso/impl/SasFileReaderImpl.java +++ b/src/main/java/com/epam/parso/impl/SasFileReaderImpl.java @@ -22,6 +22,7 @@ import com.epam.parso.Column; import com.epam.parso.SasFileProperties; import com.epam.parso.SasFileReader; +import com.epam.parso.date.OutputDateType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -80,6 +81,21 @@ public SasFileReaderImpl(InputStream inputStream, String encoding) { sasFileParser = new SasFileParser.Builder(inputStream).encoding(encoding).build(); } + /** + * Builds an instance of SasFileReaderImpl from the file contained in the input stream + * with the specified encoding and type of output date formatting. + * Reads only metadata (properties and column information) of the sas7bdat file. + * + * @param inputStream an input stream which should contain a correct sas7bdat file. + * @param encoding the string containing the encoding to use in strings output + * @param outputDateType type of formatting for date columns + */ + public SasFileReaderImpl(InputStream inputStream, String encoding, + OutputDateType outputDateType) { + sasFileParser = new SasFileParser.Builder(inputStream).encoding(encoding) + .outputDateType(outputDateType).build(); + } + /** * Builds an object of the SasFileReaderImpl class from the file contained in the input stream with a flag of * the binary or string format of the data output. diff --git a/src/test/java/com/epam/parso/BugsTest.java b/src/test/java/com/epam/parso/BugsTest.java index 4af49b6..6e2ab4a 100644 --- a/src/test/java/com/epam/parso/BugsTest.java +++ b/src/test/java/com/epam/parso/BugsTest.java @@ -25,9 +25,6 @@ import java.io.InputStream; import java.nio.file.Files; import java.nio.file.Paths; -import java.time.LocalDateTime; -import java.time.ZoneOffset; -import java.util.Date; import java.util.List; import static org.assertj.core.api.Assertions.assertThat; @@ -218,31 +215,4 @@ public void testInfinityLoopUnbufferedIssue58() throws Exception { assertThat(sasFileReader.getSasFileProperties().getRowCount()).isEqualTo(0); } } - - /** - * Converts year, month and day to UTC Date. - */ - private static Date dateOf(int year, int month, int day) { - return Date.from(LocalDateTime.of(year, month, day, 0, 0).toInstant(ZoneOffset.UTC)); - } - - @Test - public void testLeapDayFixIssue81() throws Exception { - try (InputStream is = this.getClass().getResourceAsStream("/bugs/81-dates.sas7bdat")) { - SasFileReader sasFileReader = new SasFileReaderImpl(is); - - Object[][] result = sasFileReader.readAll(); - assertThat(result.length).isEqualTo(10); - assertThat(result[0][1]).isEqualTo(dateOf(9999, 12, 31)); - assertThat(result[1][1]).isEqualTo(dateOf(2049, 12, 31)); - assertThat(result[2][1]).isEqualTo(dateOf(2099, 12, 31)); - assertThat(result[3][1]).isEqualTo(dateOf(4000, 2, 28)); - assertThat(result[4][1]).isEqualTo(dateOf(4000, 3, 1)); - assertThat(result[5][1]).isEqualTo(dateOf(4000, 12, 31)); - assertThat(result[6][1]).isEqualTo(dateOf(8000, 2, 28)); - assertThat(result[7][1]).isEqualTo(dateOf(8000, 3, 1)); - assertThat(result[8][1]).isEqualTo(dateOf(8000, 12, 31)); - assertThat(result[9][1]).isEqualTo(dateOf(8001, 2, 21)); - } - } } diff --git a/src/test/java/com/epam/parso/SasDateTypeTest.java b/src/test/java/com/epam/parso/SasDateTypeTest.java new file mode 100644 index 0000000..6a209b3 --- /dev/null +++ b/src/test/java/com/epam/parso/SasDateTypeTest.java @@ -0,0 +1,209 @@ +package com.epam.parso; + +import com.epam.parso.impl.SasFileReaderImpl; +import org.junit.Test; + +import java.io.InputStream; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.util.Date; + +import static com.epam.parso.date.OutputDateType.*; +import static org.junit.Assert.assertEquals; + +/** + * These tests cover JAVA_DATE, JAVA_TEMPORAL, EPOCH_SECONDS and SAS_VALUE. + *

+ * These tests don't test SAS_FORMAT or SAS_FORMAT_TRIM, all SAS_FORMAT-related + * cases are covered by separate Sas[FormatName]Test classes. + */ +public class SasDateTypeTest { + + private static Date utcDateOf(int year, int month, int day) { + return Date.from(LocalDateTime.of(year, month, day, 0, 0).toInstant(ZoneOffset.UTC)); + } + + private static Date utcDateOf(int year, int month, int day, int hour, int minute, int second, int nanos) { + return Date.from(LocalDateTime.of(year, month, day, hour, minute, second, nanos).toInstant(ZoneOffset.UTC)); + } + + private static double utcEpochSecondsOf(int year, int month, int day) { + return LocalDateTime.of(year, month, day, 0, 0).toEpochSecond(ZoneOffset.UTC); + } + + @Test + public void testJavaDate() throws Exception { + try (InputStream is = getClass().getResourceAsStream("/dates/sas7bdat/date_format_date.sas7bdat")) { + SasFileReader sasFileReader = new SasFileReaderImpl(is, null, JAVA_DATE_LEGACY); + Object[][] result = sasFileReader.readAll(); + assertEquals(utcDateOf(2013, 3, 17), result[0][0]); + } + try (InputStream is = getClass().getResourceAsStream("/dates/sas7bdat/date_format_datetime.sas7bdat")) { + SasFileReader sasFileReader = new SasFileReaderImpl(is, null, JAVA_DATE_LEGACY); + Object[][] result = sasFileReader.readAll(); + assertEquals(utcDateOf(2013, 3, 17, 19, 53, 1, 321000000), result[0][0]); + } + try (InputStream is = getClass().getResourceAsStream("/dates/sas7bdat/date_format_time.sas7bdat")) { + SasFileReader sasFileReader = new SasFileReaderImpl(is, null, JAVA_DATE_LEGACY); + Object[][] result = sasFileReader.readAll(); + assertEquals(71581.321, result[0][0]); + } + } + + @Test + public void testJavaTemporal() throws Exception { + try (InputStream is = getClass().getResourceAsStream("/dates/sas7bdat/date_format_date.sas7bdat")) { + SasFileReader sasFileReader = new SasFileReaderImpl(is, null, JAVA_TEMPORAL); + Object[][] result = sasFileReader.readAll(); + assertEquals(result[0][0], LocalDate.of(2013, 3, 17)); + } + try (InputStream is = getClass().getResourceAsStream("/dates/sas7bdat/date_format_datetime.sas7bdat")) { + SasFileReader sasFileReader = new SasFileReaderImpl(is, null, JAVA_TEMPORAL); + Object[][] result = sasFileReader.readAll(); + assertEquals(LocalDateTime.of(2013, 3, 17, 19, 53, 1, 321000000), result[0][0]); + } + try (InputStream is = getClass().getResourceAsStream("/dates/sas7bdat/date_format_time.sas7bdat")) { + SasFileReader sasFileReader = new SasFileReaderImpl(is, null, JAVA_TEMPORAL); + Object[][] result = sasFileReader.readAll(); + assertEquals(71581.321, result[0][0]); + } + } + + @Test + public void testSasValue() throws Exception { + try (InputStream is = getClass().getResourceAsStream("/dates/sas7bdat/date_format_date.sas7bdat")) { + SasFileReader sasFileReader = new SasFileReaderImpl(is, null, SAS_VALUE); + Object[][] result = sasFileReader.readAll(); + assertEquals(19434.0, result[0][0]); + } + try (InputStream is = getClass().getResourceAsStream("/dates/sas7bdat/date_format_datetime.sas7bdat")) { + SasFileReader sasFileReader = new SasFileReaderImpl(is, null, SAS_VALUE); + Object[][] result = sasFileReader.readAll(); + assertEquals(1679169181.321, result[0][0]); + } + try (InputStream is = getClass().getResourceAsStream("/dates/sas7bdat/date_format_time.sas7bdat")) { + SasFileReader sasFileReader = new SasFileReaderImpl(is, null, SAS_VALUE); + Object[][] result = sasFileReader.readAll(); + assertEquals(71581.321, result[0][0]); + } + } + + @Test + public void testEpochSeconds() throws Exception { + try (InputStream is = getClass().getResourceAsStream("/dates/sas7bdat/date_format_date.sas7bdat")) { + SasFileReader sasFileReader = new SasFileReaderImpl(is, null, EPOCH_SECONDS); + Object[][] result = sasFileReader.readAll(); + assertEquals(1363478400.0, result[0][0]); + } + try (InputStream is = getClass().getResourceAsStream("/dates/sas7bdat/date_format_datetime.sas7bdat")) { + SasFileReader sasFileReader = new SasFileReaderImpl(is, null, EPOCH_SECONDS); + Object[][] result = sasFileReader.readAll(); + assertEquals(1363549981.321, result[0][0]); + } + try (InputStream is = getClass().getResourceAsStream("/dates/sas7bdat/date_format_time.sas7bdat")) { + SasFileReader sasFileReader = new SasFileReaderImpl(is, null, EPOCH_SECONDS); + Object[][] result = sasFileReader.readAll(); + assertEquals(71581.321, result[0][0]); + } + } + + @Test + public void testLeapJavaDate() throws Exception { + try (InputStream is = getClass().getResourceAsStream("/dates/sas7bdat/date_leap_days.sas7bdat")) { + SasFileReader sasFileReader = new SasFileReaderImpl(is, null, JAVA_DATE_LEGACY); + + Object[][] result = sasFileReader.readAll(); + assertEquals(16, result.length); + + Date[] dates = { + utcDateOf(2000, 2, 28), + utcDateOf(2000, 2, 29), + utcDateOf(2000, 3, 1), + utcDateOf(2000, 12, 31), + utcDateOf(4000, 2, 28), + utcDateOf(4000, 3, 1), + utcDateOf(4000, 12, 31), + utcDateOf(6000, 2, 28), + utcDateOf(6000, 2, 29), + utcDateOf(6000, 3, 1), + utcDateOf(6000, 12, 31), + utcDateOf(8000, 2, 28), + utcDateOf(8000, 3, 1), + utcDateOf(8000, 12, 31), + null, + utcDateOf(9999, 12, 31), + }; + for (int i = 0; i < dates.length; i++) { + assertEquals(dates[i], result[i][0]); + assertEquals(dates[i], result[i][1]); + } + } + } + + @Test + public void testLeapEpochSeconds() throws Exception { + try (InputStream is = getClass().getResourceAsStream("/dates/sas7bdat/date_leap_days.sas7bdat")) { + SasFileReader sasFileReader = new SasFileReaderImpl(is, null, EPOCH_SECONDS); + + Object[][] result = sasFileReader.readAll(); + assertEquals(16, result.length); + + Double[] dates = { + utcEpochSecondsOf(2000, 2, 28), + utcEpochSecondsOf(2000, 2, 29), + utcEpochSecondsOf(2000, 3, 1), + utcEpochSecondsOf(2000, 12, 31), + utcEpochSecondsOf(4000, 2, 28), + utcEpochSecondsOf(4000, 3, 1), + utcEpochSecondsOf(4000, 12, 31), + utcEpochSecondsOf(6000, 2, 28), + utcEpochSecondsOf(6000, 2, 29), + utcEpochSecondsOf(6000, 3, 1), + utcEpochSecondsOf(6000, 12, 31), + utcEpochSecondsOf(8000, 2, 28), + utcEpochSecondsOf(8000, 3, 1), + utcEpochSecondsOf(8000, 12, 31), + null, + utcEpochSecondsOf(9999, 12, 31), + }; + for (int i = 0; i < dates.length; i++) { + assertEquals(dates[i], result[i][0]); + assertEquals(dates[i], result[i][1]); + } + } + } + + @Test + public void testLeapSasValue() throws Exception { + try (InputStream is = getClass().getResourceAsStream("/dates/sas7bdat/date_leap_days.sas7bdat")) { + SasFileReader sasFileReader = new SasFileReaderImpl(is, null, SAS_VALUE); + + Object[][] result = sasFileReader.readAll(); + assertEquals(16, result.length); + + Double[][] dates = { + {14668D, 1267315200D}, + {14669D, 1267401600D}, + {14670D, 1267488000D}, + {14975D, 1293840000D}, + {745153D, 64381219200D}, + {745154D, 64381305600D}, + {745459D, 64407657600D}, + {1475637D, 127495036800D}, + {1475638D, 127495123200D}, + {1475639D, 127495209600D}, + {1475944D, 127521561600D}, + {2206122D, 190608940800D}, + {2206123D, 190609027200D}, + {2206428D, 190635379200D}, + {Double.NaN, Double.NaN}, + {2936547D, 253717660800D}, + }; + for (int i = 0; i < dates.length; i++) { + assertEquals(dates[i][0], result[i][0]); + assertEquals(dates[i][1], result[i][1]); + } + } + } +} diff --git a/src/test/resources/bugs/81-dates.sas7bdat b/src/test/resources/bugs/81-dates.sas7bdat deleted file mode 100644 index 86f53bf8ce255ef449b165967c92d9b02c06f94a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 131072 zcmeI&&ubi2902gQNwjHmK~(<@eI7JyVV@ z?^!vYk8Xc_)h)yX2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5cqEiEbm!4zu6JJ^lQv^)k9x4 z{O(2>k92odFU9;&XF9xgy-XbK>YUgW^M_NqRt{FLmHiWy{AV%$Y$gA8%uiPG&&B*y z%E$iZ@8#fj`BWYcCx)xfXM@EzUOm`~iK(K0_)6J_-lAW+`csStivHZCeX+h?#P6=h zLq+`K7ioT15wCnZ73&*$yx5M1i}{l)X?|}JzrP-5)fxc;1PBlyK!5-N0t5&U_$LAb zAvBsHG_!szdtDg$XlU$g2uBv)dOhFWFS|e9pZBW+%bz#1yheZk0RjXF5FkK+009C7 z2;2t(`SXIkAslOL{=DEm5O9hF2oNAZfB*pk1PBlyK!8A1Ab(y^FaN*b!lNOqRoaoU z^8X4_U)@|xfB*pk1PBlyK!5-N0t5)$n*xm;<@18Ytgk#SSX*qB&kIt2Zwfdo0t5&U zAaHjCrd|kPFfQF#><&v=eC1;IEx*SDhsxjM)Zd+>wJHGu1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UxR(TO{;^iH>N)MKCSp7hdOBuqw|z&^7QxMO`L*Hj z_1V1au~^#`x!$twuH`vvF@8_teTh#l4-|#!c6T^4g;9&DP1= zNT{CmpRxErF<-pZ8TwP1(x@$oZqtG4ky$1EPZ|{PWQMJr_V9P^|=2O7x&aVT=Vah^=|krsjuI& iL+CHYyEbID+HX&G{nc){bDxM;VF;UC*Xh)X-TnmwQ diff --git a/src/test/resources/csv/dates_leap_days.csv b/src/test/resources/csv/dates_leap_days.csv deleted file mode 100644 index fbf8af3..0000000 --- a/src/test/resources/csv/dates_leap_days.csv +++ /dev/null @@ -1,15 +0,0 @@ -d, dt -28Feb2000,28Feb2000:00:00:00.00 -29Feb2000,29Feb2000:00:00:00.00 -01Mar2000,01Mar2000:00:00:00.00 -31Dec2000,31Dec2000:00:00:00.00 -28Feb4000,28Feb4000:00:00:00.00 -01Mar4000,01Mar4000:00:00:00.00 -31Dec4000,31Dec4000:00:00:00.00 -28Feb6000,28Feb6000:00:00:00.00 -29Feb6000,29Feb6000:00:00:00.00 -01Mar6000,01Mar6000:00:00:00.00 -31Dec6000,31Dec6000:00:00:00.00 -28Feb8000,28Feb8000:00:00:00.00 -01Mar8000,01Mar8000:00:00:00.00 -31Dec8000,31Dec8000:00:00:00.00 diff --git a/src/test/resources/csv/dates_leap_days_meta.csv b/src/test/resources/csv/dates_leap_days_meta.csv deleted file mode 100644 index 9e18b06..0000000 --- a/src/test/resources/csv/dates_leap_days_meta.csv +++ /dev/null @@ -1,23 +0,0 @@ -Number,Name,Type,Data length,Format,Label -1,d,Numeric,8,DATE9., -2,dt,Numeric,8,DATETIME20., -Bitness: x64 -Compressed: null -Endianness: LITTLE_ENDIANNESS -Encoding: ISO-8859-1 -Name: DATES_LEAP_DAYS -File type: DATA -File label: Leap days dataset -Date created: Fri Jan 01 13:53:59 MSK 2021 -Date modified: Fri Jan 01 13:53:59 MSK 2021 -SAS release: 9.0401M5 -SAS server type: Linux -OS name: x86_64 -OS type: 3.10.0-1160.2.1. -Header Length: 4096 -Page Length: 4096 -Page Count: 1 -Row Length: 16 -Row Count: 14 -Mix Page Row Count: 124 -Columns Count: 2 diff --git a/src/test/resources/dates/sas7bdat/date_format_date.sas7bdat b/src/test/resources/dates/sas7bdat/date_format_date.sas7bdat new file mode 100644 index 0000000000000000000000000000000000000000..9d3b927fd6ed332db3cb491278ff98b38358ba2b GIT binary patch literal 8192 zcmeHMzi-n(6uvYmO=%<=$k2fbtL+e^V%azi=@z4=DpIOeBtwTvVF0NthykPw3}x#8 z3(AOKW8)JcJ6eEl>vctC%Nyv@80)(_xbFLYeJ&- z;p^jDZwmQO@1AAJ^hfu}+@;yPw?I5EHQ}bhv~+3!_fm@&bJK6Om~TlfEU9rQ$7dAQ{r%8d`tthNN2^d~DMF8M5RkBHJ0-{6T&dr^_plu! zdA94=&K1`!JN8xEwPRh}ZCA>zaw$UDdEY+W^6`jzg!l6L;T`5_$0rj^Y!omG7zK<1 zMggOMQNSo*6fg=H1&jhlfxoE03wUppPx&eX$V8sKR+FQgh>PjL%=duL4Z>vatN?Z< z20OabBJLmzFYIl=cP0EdD}d(`-T{0=!kd6s5eHk);N2;aAm?{9cvFKHGg(-kiLcWGhiZWkZ{-8vT_>`A{m?blGKSULS zpdXwDC&3h%B)x!ycyyK%sB;iO+~XB8Q6-Go?;O@J#@{p;Tf8^wFG{f^#T6-rkQ*3U zun~ewpu#ep4{H^$Yhbs*?tPo zn13zivpHeTYRq@(bR#BrW-439dxeuuXn)Ek~f$ghNXqw4%G{Xb}9W99c~Y|uYxW8pWz&d2(g zn|BsCQssZu?}h*?b@5qr858iK0r3T}xK7Ywy0OC*(7lfiXQ{n-pW*8##^)dv~91d&ogZjuE94Z6%Z* zB19r%r@Zv#1u!si{1hGZYrdj9h`^}roWG8v(rQ-KN@|)l9 z&2RR-&+Kk;drH~k)1O~m`e=0ct54o6s(~-7Z;Txq8m=E#^?KA-jk0O|kyEcnkJZ&t zrAqzTl=GaHOD{i)1i>IG?cCOi8Ovde*xmIb*ku4b*=$>XWJvn`8fFiM=g(bSky;Y!~&(2Au?}$3Ahu>UI^ND_U<_)Qj zxn;F{DJ?AYxhHO}r1@Ks)Fqare%(x$`j4-qg{QswYf@kG=EqW>Hg$=GKz;pUI>Dd6 z7O2O8dLd9>zYthIP>%!kLZH6>a$x;HJr2|hf%^JOf%OCRI8ZO_Qcqt+DYMP->+N@6 z+w*Pneu?J#hzKBn00IagfB*srAbX3dO=!p8| z*xR+FG@z1CM4~#R_N(OGb*FP%_ulJlcJ```j#B&7-OfS1y8Fr7ko70b!$oyxa&&zP zQFb-mycp@#)5GNE*|sJxdFLm*yyE3kUOw&RXT1EZmoIqvl9w-g`FSswN9(5dsqb&q zJAcK?CG7^G>s-rh@0)^e7kj4$5wl0uI0z4^m2YJ*+0K_szeIwgmL*K}@n!XXgR_k#ewI26T>v12oE%#w}Mvvq} zX!fQcCqv&cPVYZR24!LM**9;Ay|R{{oA-{Xo9pv)^La4!J>GgI-;{}Fp1Dsymw&zV zch}CxU$WlLvy}QX=e*t4`LCS+T_4G{y5IOp{h^Pf{%i6}ALD7reh*iUcgI(yV)iAQ zAOnA0^S9}JGjY`n()hWg=6zzKJButN5PBYcXnv?}_R~Cm-sF!;gUOHVk{3fS@UQDd rWxivS8qNL(QEIR|9^X>g>7@$#KcqzGAR6lP%a-uvc#Z+73zzRB(~ z=C>CgZZ1E~q+dU~)6WK8*=r*whSE-sIZmR-PAIqDLzr)Fx^ z$x5TTG&MIrTWKscW@f8d(o{%|!ub_J5#942@Ah4MvQ*jB+hW7|cmFj}w+fi6#Nm5NKnF&`D<9)H+b!Nc#7S}Hj=B{c2%QG&5e z222J_222J_222J_222J_222J_222J_2L56O4#IOQ9rAV$AU(q@!_{{wY3F;xp3eav z3B#;;Kmv;x2m03sr1BE|;ekC5_$t9`{Sw$Bc)L#m4T5g~UPc^bOM{m*czeGj(l{F$ zyrscQ8oa$v6JLY3GRQ$1+d_^Eip7Q1Lcgt+ro3ax4r&yZ5Ll55t z+iu&H>uQj7++BCa?c*Mf62A*!eW5ny7;!v5YU4UU9P16W_rMWxyiRJv zHsp4@${mQ;&F^bWzix5y}d*?$Zz5OfgL%c_C~Ri d#h6C}$j^pJ^4&TS*)fhN3SOrI{6{_y`VJ27n@#`# literal 0 HcmV?d00001 diff --git a/src/test/resources/sas7bdat/dates_leap_days.sas7bdat b/src/test/resources/dates/sas7bdat/date_leap_days.sas7bdat similarity index 82% rename from src/test/resources/sas7bdat/dates_leap_days.sas7bdat rename to src/test/resources/dates/sas7bdat/date_leap_days.sas7bdat index bd701baa17fb9dea919fdb1ddcbddf988b6a0964..dc8c5fbce5be44a061d152ba8aaad7d8293d2ad9 100644 GIT binary patch delta 390 zcmZp0XmFU2FfmbjVzn-xUdd7A(_8L1LgHzzXA z=bJczlf?umF|kpE`x+Zm2b0ufWdUD987>Il2P(|K2w{BSgwQ%@;#(#!6cCrJgUXtr z$=Xf62$cQD0TFh9$wKLWlNkm7NdNo)AIyi)4Tb4$jtmSAN(r65PC(Z_W1IX+!hq3W zvb3Z=j#RZ3aJCpED^AtkZ07)X>P7N?c~4V&yCrNQX4xln346C>ZmM)AoC vvIdMbn*(L-8NnW%oCq}hhMWbX#AHGFd}b8}!O6|?!Q5?d_wY@AC@&8HQKDO$ diff --git a/src/test/resources/sas7bdat/dates_leap_days.sas b/src/test/resources/sas7bdat/dates_leap_days.sas deleted file mode 100644 index a3cbb33..0000000 --- a/src/test/resources/sas7bdat/dates_leap_days.sas +++ /dev/null @@ -1,57 +0,0 @@ -/* - SAS program to generate sas7bdat file with two types of columns: date and datetime. - Both columns contain data around leap days. - Years 4000 and 8000 don't have leap days in terms of SAS. - Years 2000 and 6000 have it. - All of them necessary for unit tests. -*/ - -options bufsize=4096 pagesize=15; - -data dev.dates_leap_days(label='Leap days dataset'); - format d date9.; - format dt datetime20.; - - d='28FEB2000'd; - dt='28FEB2000:00:00:00'dt; - output; - d='29FEB2000'd; - dt='29FEB2000:00:00:00'dt; - output; - d='01MAR2000'd; - dt='01MAR2000:00:00:00'dt; - output; - d='31DEC2000'd; - dt='31DEC2000:00:00:00'dt; - output; - d='28FEB4000'd; - dt='28FEB4000:00:00:00'dt; - output; - d='01MAR4000'd; - dt='01MAR4000:00:00:00'dt; - output; - d='31DEC4000'd; - dt='31DEC4000:00:00:00'dt; - output; - d='28FEB6000'd; - dt='28FEB6000:00:00:00'dt; - output; - d='29FEB6000'd; - dt='29FEB6000:00:00:00'dt; - output; - d='01MAR6000'd; - dt='01MAR6000:00:00:00'dt; - output; - d='31DEC6000'd; - dt='31DEC6000:00:00:00'dt; - output; - d='28FEB8000'd; - dt='28FEB8000:00:00:00'dt; - output; - d='01MAR8000'd; - dt='01MAR8000:00:00:00'dt; - output; - d='31DEC8000'd; - dt='31DEC8000:00:00:00'dt; - output; -run;