Merge pull request pentaho#5276 from denisprotopopov/PDI-17117

[PDI-17117]-"Text file input" step picks up the last file in order when supplies with 2 files of same names

engine/src/main/java/org/pentaho/di/trans/steps/file/BaseFileInputStep.java

@@ -2,7 +2,7 @@
  *
  * Pentaho Data Integration
  *
- * Copyright (C) 2002-2017 by Hitachi Vantara : http://www.pentaho.com
+ * Copyright (C) 2002-2018 by Hitachi Vantara : http://www.pentaho.com
  *
  *******************************************************************************
  *
@@ -152,7 +152,9 @@ protected boolean openNextFile() {
 
       fillFileAdditionalFields( data, data.file );
       if ( meta.inputFiles.passingThruFields ) {
-        data.currentPassThruFieldsRow = data.passThruFields.get( data.file );
+        StringBuilder sb = new StringBuilder();
+        sb.append( data.currentFileIndex ).append( "_" ).append( data.file );
+        data.currentPassThruFieldsRow = data.passThruFields.get( sb.toString() );
       }
 
       // Add this files to the result of this transformation.
@@ -305,7 +307,7 @@ private RowMetaInterface[] filesFromPreviousStep() throws KettleException {
       RowMetaInterface prevInfoFields = rowSet.getRowMeta();
       if ( idx < 0 ) {
         if ( meta.inputFiles.passingThruFields ) {
-          data.passThruFields = new HashMap<FileObject, Object[]>();
+          data.passThruFields = new HashMap<String, Object[]>();
           infoStep = new RowMetaInterface[] { prevInfoFields };
           data.nrPassThruFields = prevInfoFields.size();
         }
@@ -323,7 +325,9 @@ private RowMetaInterface[] filesFromPreviousStep() throws KettleException {
         FileObject fileObject = KettleVFS.getFileObject( fileValue, getTransMeta() );
         data.files.addFile( fileObject );
         if ( meta.inputFiles.passingThruFields ) {
-          data.passThruFields.put( fileObject, fileRow );
+          StringBuilder sb = new StringBuilder();
+          sb.append( data.files.nrOfFiles() > 0 ? data.files.nrOfFiles() - 1 : 0 ).append( "_" ).append( fileObject.toString() );
+          data.passThruFields.put( sb.toString(), fileRow );
         }
       } catch ( KettleFileException e ) {
         logError( BaseMessages.getString( PKG, "TextFileInput.Log.Error.UnableToCreateFileObject", fileValue ), e );

engine/src/main/java/org/pentaho/di/trans/steps/file/BaseFileInputStepData.java

@@ -2,7 +2,7 @@
  *
  * Pentaho Data Integration
  *
- * Copyright (C) 2017 by Hitachi Vantara : http://www.pentaho.com
+ * Copyright (C) 2017-2018 by Hitachi Vantara : http://www.pentaho.com
  *
  *******************************************************************************
  *
@@ -54,7 +54,7 @@ public class BaseFileInputStepData extends BaseStepData {
 
   public RowMetaInterface outputRowMeta;
 
-  public HashMap<FileObject, Object[]> passThruFields;
+  public HashMap<String, Object[]> passThruFields;
 
   public Object[] currentPassThruFieldsRow;
 

engine/src/test/java/org/pentaho/di/trans/steps/fileinput/text/TextFileInputTest.java

@@ -29,11 +29,14 @@
 import org.junit.Test;
 import org.mockito.Mockito;
 import org.pentaho.di.core.KettleEnvironment;
+import org.pentaho.di.core.RowSet;
 import org.pentaho.di.core.exception.KettleFileException;
 import org.pentaho.di.core.fileinput.FileInputList;
 import org.pentaho.di.core.playlist.FilePlayListAll;
 import org.pentaho.di.core.row.RowMeta;
+import org.pentaho.di.core.row.RowMetaInterface;
 import org.pentaho.di.core.row.value.ValueMetaString;
+import org.pentaho.di.core.util.Assert;
 import org.pentaho.di.core.variables.Variables;
 import org.pentaho.di.core.vfs.KettleVFS;
 import org.pentaho.di.junit.rules.RestorePDIEngineEnvironment;
@@ -54,6 +57,7 @@
 import java.io.InputStreamReader;
 import java.io.OutputStream;
 import java.io.UnsupportedEncodingException;
+import java.util.Iterator;
 import java.util.List;
 
 import static org.junit.Assert.assertEquals;
@@ -251,6 +255,42 @@ public void testHandleOpenFileException() throws Exception {
     assertEquals( 0, textFileInput.getErrors() );
   }
 
+  @Test
+  public void test_PDI17117() throws Exception {
+    final String virtualFile = createVirtualFile( "pdi-14832.txt", "1,\n" );
+
+    BaseFileField col2 = field( "col2" );
+    col2.setIfNullValue( "DEFAULT" );
+
+    TextFileInputMeta meta = createMetaObject( field( "col1" ), col2 );
+
+    meta.inputFiles.passingThruFields = true;
+    meta.inputFiles.acceptingFilenames = true;
+    TextFileInputData data = createDataObject( virtualFile, ",", "col1", "col2" );
+
+    TextFileInput input = Mockito.spy( StepMockUtil.getStep( TextFileInput.class, TextFileInputMeta.class, "test" ) );
+
+    RowSet rowset = Mockito.mock( RowSet.class );
+    RowMetaInterface rwi = Mockito.mock( RowMetaInterface.class );
+    Object[] obj1 = new Object[2];
+    Object[] obj2 = new Object[2];
+    Mockito.doReturn( rowset ).when( input ).findInputRowSet( null );
+    Mockito.doReturn( null ).when( input ).getRowFrom( rowset );
+    Mockito.when( input.getRowFrom( rowset ) ).thenReturn( obj1, obj2, null );
+    Mockito.doReturn( rwi ).when( rowset ).getRowMeta();
+    Mockito.when( rwi.getString( obj2, 0 ) ).thenReturn( "filename1", "filename2" );
+    List<Object[]> output = TransTestingUtil.execute( input, meta, data, 0, false );
+    Iterator iterator = data.passThruFields.keySet().iterator();
+    String s = (String) iterator.next();
+    Assert.assertTrue( s.startsWith( "0_file" ) );
+    Assert.assertTrue( s.endsWith( "filename1" ) );
+    s = (String) iterator.next();
+    Assert.assertTrue( s.startsWith( "1_file" ) );
+    Assert.assertTrue( s.endsWith( "filename2" ) );
+
+    deleteVfsFile( virtualFile );
+  }
+
   private TextFileInputMeta createMetaObject( BaseFileField... fields ) {
     TextFileInputMeta meta = new TextFileInputMeta();
     meta.content.fileCompression = "None";