Skip to content

Commit

Permalink
[PDI-18369] Converting Integer to Timestamp type using Select values …
Browse files Browse the repository at this point in the history
…step results in incorrect date

[BACKLOG-33399] Big data timestamp conversion should match core PDI behavior
ValueMetaConverter should NOT match core PDI behavior by default: it handles values coming from, and going to, Avro/Parquet/Orc files and must only take in consideration those Specs!
  • Loading branch information
smmribeiro committed Mar 29, 2020
1 parent a2d1d79 commit 9ad64ce
Show file tree
Hide file tree
Showing 2 changed files with 7 additions and 58 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -39,11 +39,13 @@
* It was initially implemented for Orc storage in the pentaho-hadoop-shims project. This class is added here because
* the conversions are not dependant on orc in any way.
*
* <p><b>Important note:</b><br/>This class is not intended to mimic the conversions that exist on PDI! It handles
* conversions from and to Avro/Parquet/Orc format, as so it must only take in consideration those Specs!</p>
* <p>
* Created by tkafalas on 12/8/2017.
*/
public class ValueMetaConverter implements Serializable, IValueMetaConverter {
private final String DEFAULT_DATE_FORMAT = ValueMetaBase.DEFAULT_DATE_FORMAT_MASK;
private SimpleDateFormat datePattern = new SimpleDateFormat( DEFAULT_DATE_FORMAT );
private SimpleDateFormat datePattern = new SimpleDateFormat( ValueMetaBase.DEFAULT_DATE_FORMAT_MASK );
private int precision = 0;

public SimpleDateFormat getDatePattern() {
Expand Down Expand Up @@ -282,8 +284,7 @@ protected Object convertFromIntegerMetaInterface( int targetValueMetaType, Objec
case ValueMetaInterface.TYPE_DATE:
return new Date( (long) value );
case ValueMetaInterface.TYPE_TIMESTAMP:
ValueMetaTimestamp pentahoTimeStamp = new ValueMetaTimestamp();
return pentahoTimeStamp.convertIntegerToTimestamp( (Long) value );
return new Timestamp( (long) value );
default:
throwBadConversionCombination( ValueMetaInterface.TYPE_INTEGER, targetValueMetaType, value );
}
Expand Down Expand Up @@ -344,15 +345,14 @@ protected Object convertFromTimestampMetaInterface( int targetValueMetaType, Obj
+ value + "'." );
}

ValueMetaTimestamp pentahoTimeStamp = new ValueMetaTimestamp();
Date dateValue;
try {
switch ( targetValueMetaType ) {
case ValueMetaInterface.TYPE_STRING:
dateValue = new Date( ( (Timestamp) value ).getTime() );
return datePattern.format( dateValue );
case ValueMetaInterface.TYPE_INTEGER:
return pentahoTimeStamp.getInteger( value );
return ( (Timestamp) value ).getTime();
case ValueMetaInterface.TYPE_TIMESTAMP:
return new Timestamp( ( (Timestamp) value ).getTime() );
case ValueMetaInterface.TYPE_DATE:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@
package org.pentaho.di.core.row.value;

import org.junit.Test;
import org.pentaho.di.core.Const;
import org.pentaho.di.core.row.ValueMetaInterface;

import java.math.BigDecimal;
Expand All @@ -33,7 +32,6 @@
import java.util.Date;
import java.util.HashMap;
import java.util.Map;
import java.util.TimeZone;

import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
Expand All @@ -55,11 +53,6 @@ public class ValueMetaConverterTest {
public void convertFromSourceToTargetDataTypeTest() throws Exception {
//"-", "Number", "String", "Date", "Boolean", "Integer", "BigNumber", "Serializable", "Binary", "Timestamp",
// "Internet Address", }

// Make sure the default conversion mode is used
System.setProperty( Const.KETTLE_TIMESTAMP_NUMBER_CONVERSION_MODE,
Const.KETTLE_TIMESTAMP_NUMBER_CONVERSION_MODE_DEFAULT );

DateFormat dateFormat = new SimpleDateFormat( "yyyy/MM/dd HH:mm:ss.SSS" );
Date date1 = ( dateFormat.parse( "1999/12/31 00:00:00.000" ) );
Date timeStamp1 = new Timestamp( dateFormat.parse( "2001/11/01 20:30:15.123" ).getTime() );
Expand Down Expand Up @@ -103,7 +96,7 @@ public void convertFromSourceToTargetDataTypeTest() throws Exception {
{ ValueMetaInterface.TYPE_INTEGER, ValueMetaInterface.TYPE_INTEGER, 1234L, 1234L },
{ ValueMetaInterface.TYPE_INTEGER, ValueMetaInterface.TYPE_NUMBER, 1234L, 1234.0 },
{ ValueMetaInterface.TYPE_INTEGER, ValueMetaInterface.TYPE_BIGNUMBER, 1234L, new BigDecimal( "1234" ) },
{ ValueMetaInterface.TYPE_INTEGER, ValueMetaInterface.TYPE_TIMESTAMP, 1000000*timeStamp1.getTime(), timeStamp1 },
{ ValueMetaInterface.TYPE_INTEGER, ValueMetaInterface.TYPE_TIMESTAMP, timeStamp1.getTime(), timeStamp1 },

{ ValueMetaInterface.TYPE_BIGNUMBER, ValueMetaInterface.TYPE_NONE, new BigDecimal( "123456.123456" ), null },
{ ValueMetaInterface.TYPE_BIGNUMBER, ValueMetaInterface.TYPE_STRING, new BigDecimal( "123456.123456" ),
Expand Down Expand Up @@ -190,48 +183,4 @@ public void convertFromSourceToTargetDataTypeTest() throws Exception {
private String getKey( int sourceType, int targetType ) {
return "" + sourceType + "," + targetType;
}

@Test
public void testTimestampIntegerConversions() throws Exception {

TimeZone.setDefault( TimeZone.getTimeZone( "Europe/London" ) );

Timestamp timestampWithMilliseconds = Timestamp.valueOf( "2019-09-01 04:34:56.12300000" );
long timestampAsMilliseconds = 1567308896123L;
Timestamp timestampWithNanoseconds = Timestamp.valueOf( "2019-09-01 04:34:56.123456789" );
long timestampAsNanoseconds = 1567308896123456789L;

Object[][] timestampTests = new Object[][] {
{ Const.KETTLE_TIMESTAMP_NUMBER_CONVERSION_MODE_LEGACY, ValueMetaInterface.TYPE_INTEGER,
ValueMetaInterface.TYPE_TIMESTAMP, timestampAsNanoseconds, timestampWithNanoseconds },
{ Const.KETTLE_TIMESTAMP_NUMBER_CONVERSION_MODE_LEGACY, ValueMetaInterface.TYPE_TIMESTAMP,
ValueMetaInterface.TYPE_INTEGER, timestampWithMilliseconds, timestampAsMilliseconds },
{ Const.KETTLE_TIMESTAMP_NUMBER_CONVERSION_MODE_MILLISECONDS, ValueMetaInterface.TYPE_INTEGER,
ValueMetaInterface.TYPE_TIMESTAMP, timestampAsMilliseconds, timestampWithMilliseconds },
{ Const.KETTLE_TIMESTAMP_NUMBER_CONVERSION_MODE_MILLISECONDS, ValueMetaInterface.TYPE_TIMESTAMP,
ValueMetaInterface.TYPE_INTEGER, timestampWithMilliseconds, timestampAsMilliseconds },
{ Const.KETTLE_TIMESTAMP_NUMBER_CONVERSION_MODE_NANOSECONDS, ValueMetaInterface.TYPE_INTEGER,
ValueMetaInterface.TYPE_TIMESTAMP, timestampAsNanoseconds, timestampWithNanoseconds },
{ Const.KETTLE_TIMESTAMP_NUMBER_CONVERSION_MODE_NANOSECONDS, ValueMetaInterface.TYPE_TIMESTAMP,
ValueMetaInterface.TYPE_INTEGER, timestampWithNanoseconds, timestampAsNanoseconds }
};

ValueMetaConverter converter = new ValueMetaConverter();

for ( Object[] test : timestampTests ) {
// Set the conversion mode
System.setProperty( Const.KETTLE_TIMESTAMP_NUMBER_CONVERSION_MODE, (String) test[ 0 ] );

Object targetValue = converter
.convertFromSourceToTargetDataType( (Integer) test[ 1 ], (Integer) test[ 2 ], test[ 3 ] );

if ( IS_VERBOSE ) {
System.out.println(
"(" + test[ 0 ] + ") type " + test[ 1 ] + "/" + test[ 2 ] + ":" + test[ 3 ].toString() + "=" + targetValue
.toString() );
}

assertEquals( test[ 4 ], targetValue );
}
}
}

0 comments on commit 9ad64ce

Please sign in to comment.