Skip to content

Commit

Permalink
Merge pull request pentaho#5286 from rmansoor/BACKLOG-22576
Browse files Browse the repository at this point in the history 
 [BACKLOG-22576] - Remove secret and access key and use the aws provider chain to retrieve access info
  • Loading branch information
@rmansoor
rmansoor authored Apr 22, 2018
2 parents e3fecdb + 5bc0e4b commit 656fbb7
Show file tree
Hide file tree
Showing 5 changed files with 67 additions and 293 deletions.
72 changes: 0 additions & 72 deletions ...s3csvinput/core/src/main/java/org/pentaho/di/trans/steps/s3csvinput/S3CsvInputDialog.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,16 +72,13 @@
import org.pentaho.di.ui.core.widget.ComboValuesSelectionListener;
import org.pentaho.di.ui.core.widget.TableView;
import org.pentaho.di.ui.core.widget.TextVar;
import org.pentaho.di.ui.core.widget.PasswordTextVar;
import org.pentaho.di.ui.trans.dialog.TransPreviewProgressDialog;
import org.pentaho.di.ui.trans.step.BaseStepDialog;
import org.pentaho.di.ui.trans.steps.textfileinput.TextFileCSVImportProgressDialog;

public class S3CsvInputDialog extends BaseStepDialog implements StepDialogInterface {
private S3CsvInputMeta inputMeta;

private TextVar wAccessKey;
private TextVar wSecretKey;
private TextVar wBucket;
private Button wbBucket; // browse for a bucket.
private TextVar wFilename;
Expand All @@ -100,7 +97,6 @@ public class S3CsvInputDialog extends BaseStepDialog implements StepDialogInterf

private boolean isReceivingInput;
private Button wRunningInParallel;
private Button wUseAwsDefaultCredentials;

public S3CsvInputDialog( Shell parent, Object in, TransMeta tr, String sname ) {
super( parent, (BaseStepMeta) in, tr, sname );
Expand Down Expand Up @@ -154,68 +150,6 @@ public void modifyText( ModifyEvent e ) {
wStepname.setLayoutData( fdStepname );
Control lastControl = wStepname;

// Access key
Label wlAccessKey = new Label( shell, SWT.RIGHT );
wlAccessKey.setText( Messages.getString( "S3CsvInputDialog.AccessKey.Label" ) ); //$NON-NLS-1$
props.setLook( wlAccessKey );
FormData fdlAccessKey = new FormData();
fdlAccessKey.top = new FormAttachment( lastControl, margin );
fdlAccessKey.left = new FormAttachment( 0, 0 );
fdlAccessKey.right = new FormAttachment( middle, -margin );
wlAccessKey.setLayoutData( fdlAccessKey );

wAccessKey = new PasswordTextVar( transMeta, shell, SWT.SINGLE | SWT.LEFT | SWT.BORDER );

props.setLook( wAccessKey );
wAccessKey.addModifyListener( lsMod );
FormData fdAccessKey = new FormData();
fdAccessKey.top = new FormAttachment( lastControl, margin );
fdAccessKey.left = new FormAttachment( middle, 0 );
fdAccessKey.right = new FormAttachment( 100, 0 );
wAccessKey.setLayoutData( fdAccessKey );
lastControl = wAccessKey;

// Secret key
Label wlSecretKey = new Label( shell, SWT.RIGHT );
wlSecretKey.setText( Messages.getString( "S3CsvInputDialog.SecretKey.Label" ) ); //$NON-NLS-1$
props.setLook( wlSecretKey );
FormData fdlSecretKey = new FormData();
fdlSecretKey.top = new FormAttachment( lastControl, margin );
fdlSecretKey.left = new FormAttachment( 0, 0 );
fdlSecretKey.right = new FormAttachment( middle, -margin );
wlSecretKey.setLayoutData( fdlSecretKey );

wSecretKey = new PasswordTextVar( transMeta, shell, SWT.SINGLE | SWT.LEFT | SWT.BORDER );

props.setLook( wSecretKey );
wSecretKey.addModifyListener( lsMod );
FormData fdSecretKey = new FormData();
fdSecretKey.top = new FormAttachment( lastControl, margin );
fdSecretKey.left = new FormAttachment( middle, 0 );
fdSecretKey.right = new FormAttachment( 100, 0 );
wSecretKey.setLayoutData( fdSecretKey );
lastControl = wSecretKey;

// Get credentials at runtime?
//
Label wlUseAwsDefaultCredentials = new Label( shell, SWT.RIGHT );
wlUseAwsDefaultCredentials.setText( Messages.getString( "S3CsvInputDialog.GetCredentialsAtRuntime.Label" ) ); //$NON-NLS-1$
props.setLook( wlUseAwsDefaultCredentials );
FormData fdlUseAwsDefaultCredentials = new FormData();
fdlUseAwsDefaultCredentials.top = new FormAttachment( lastControl, margin );
fdlUseAwsDefaultCredentials.left = new FormAttachment( 0, 0 );
fdlUseAwsDefaultCredentials.right = new FormAttachment( middle, -margin );
wlUseAwsDefaultCredentials.setLayoutData( fdlUseAwsDefaultCredentials );
wUseAwsDefaultCredentials = new Button( shell, SWT.CHECK );
props.setLook( wUseAwsDefaultCredentials );
wUseAwsDefaultCredentials
.setToolTipText( Messages.getString( "S3CsvInputDialog.GetCredentialsAtRuntime.Tooltip" ) );
FormData fdUseAwsDefaultCredentials = new FormData();
fdUseAwsDefaultCredentials.top = new FormAttachment( lastControl, margin );
fdUseAwsDefaultCredentials.left = new FormAttachment( middle, 0 );
wUseAwsDefaultCredentials.setLayoutData( fdUseAwsDefaultCredentials );
lastControl = wUseAwsDefaultCredentials;

// Bucket name
Label wlBucket = new Label( shell, SWT.RIGHT );
wlBucket.setText( Messages.getString( "S3CsvInputDialog.Bucket.Label" ) ); //$NON-NLS-1$
Expand Down Expand Up @@ -689,9 +623,6 @@ public void getData() {
*/
public void getData( S3CsvInputMeta inputMeta ) {
wStepname.setText( stepname );
wAccessKey.setText( Const.NVL( inputMeta.getAwsAccessKey(), "" ) );
wSecretKey.setText( Const.NVL( inputMeta.getAwsSecretKey(), "" ) );
wUseAwsDefaultCredentials.setSelection( inputMeta.getUseAwsDefaultCredentials() );
wBucket.setText( Const.NVL( inputMeta.getBucket(), "" ) );

if ( isReceivingInput ) {
Expand Down Expand Up @@ -738,9 +669,6 @@ private void cancel() {

private void getInfo( S3CsvInputMeta inputMeta ) {

inputMeta.setAwsAccessKey( wAccessKey.getText() );
inputMeta.setAwsSecretKey( wSecretKey.getText() );
inputMeta.setUseAwsDefaultCredentials( wUseAwsDefaultCredentials.getSelection() );
inputMeta.setBucket( wBucket.getText() );

if ( isReceivingInput ) {
Expand Down
94 changes: 2 additions & 92 deletions ...s/s3csvinput/core/src/main/java/org/pentaho/di/trans/steps/s3csvinput/S3CsvInputMeta.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,6 @@
import org.pentaho.di.core.util.Utils;
import org.pentaho.di.core.annotations.Step;
import org.pentaho.di.core.database.DatabaseMeta;
import org.pentaho.di.core.encryption.Encr;
import org.pentaho.di.core.exception.KettleException;
import org.pentaho.di.core.exception.KettleStepException;
import org.pentaho.di.core.exception.KettleXMLException;
Expand Down Expand Up @@ -117,15 +116,6 @@ public class S3CsvInputMeta extends BaseStepMeta implements StepMetaInterface, I
@Injection( name = "RUNNING_IN_PARALLEL" )
private boolean runningInParallel;

@Injection( name = "AWS_ACCESS_KEY" )
private String awsAccessKey;

@Injection( name = "AWS_SECRET_KEY" )
private String awsSecretKey;

@Injection( name = "USE_AWS_DEFAULT_CREDENTIALS" )
private boolean useAwsDefaultCredentials;

public S3CsvInputMeta() {
super(); // allocate BaseStepMeta
allocate( 0 );
Expand Down Expand Up @@ -154,8 +144,6 @@ public void setDefault() {

private void readData( Node stepnode ) throws KettleXMLException {
try {
awsAccessKey = Encr.decryptPasswordOptionallyEncrypted( XMLHandler.getTagValue( stepnode, "aws_access_key" ) );
awsSecretKey = Encr.decryptPasswordOptionallyEncrypted( XMLHandler.getTagValue( stepnode, "aws_secret_key" ) );
bucket = XMLHandler.getTagValue( stepnode, "bucket" );
filename = XMLHandler.getTagValue( stepnode, "filename" );
filenameField = XMLHandler.getTagValue( stepnode, "filename_field" );
Expand All @@ -167,13 +155,6 @@ private void readData( Node stepnode ) throws KettleXMLException {
headerPresent = "Y".equalsIgnoreCase( XMLHandler.getTagValue( stepnode, "header" ) );
lazyConversionActive = "Y".equalsIgnoreCase( XMLHandler.getTagValue( stepnode, "lazy_conversion" ) );
runningInParallel = "Y".equalsIgnoreCase( XMLHandler.getTagValue( stepnode, "parallel" ) );
String awsDefaultCredentials = XMLHandler.getTagValue( stepnode, "use_aws_default_credentials" );
if ( awsDefaultCredentials == null || awsDefaultCredentials.equalsIgnoreCase( "N" ) ) {
this.useAwsDefaultCredentials = false;
} else {
this.useAwsDefaultCredentials = true;
}

Node fields = XMLHandler.getSubNode( stepnode, "fields" );
int nrfields = XMLHandler.countNodes( fields, "field" );

Expand Down Expand Up @@ -207,10 +188,6 @@ public void allocate( int nrFields ) {
public String getXML() {
StringBuffer retval = new StringBuffer( 500 );

retval.append( " " ).append( XMLHandler.addTagValue( "aws_access_key", Encr.encryptPasswordIfNotUsingVariables(
awsAccessKey ) ) );
retval.append( " " ).append( XMLHandler.addTagValue( "aws_secret_key", Encr.encryptPasswordIfNotUsingVariables(
awsSecretKey ) ) );
retval.append( " " ).append( XMLHandler.addTagValue( "bucket", bucket ) );
retval.append( " " ).append( XMLHandler.addTagValue( "filename", filename ) );
retval.append( " " ).append( XMLHandler.addTagValue( "filename_field", filenameField ) );
Expand All @@ -222,7 +199,6 @@ public String getXML() {
retval.append( " " ).append( XMLHandler.addTagValue( "max_line_size", maxLineSize ) );
retval.append( " " ).append( XMLHandler.addTagValue( "lazy_conversion", lazyConversionActive ) );
retval.append( " " ).append( XMLHandler.addTagValue( "parallel", runningInParallel ) );
retval.append( " " ).append( XMLHandler.addTagValue( "use_aws_default_credentials", useAwsDefaultCredentials ) );

retval.append( " <fields>" ).append( Const.CR );
for ( int i = 0; i < inputFields.length; i++ ) {
Expand Down Expand Up @@ -251,8 +227,6 @@ public String getXML() {
public void readRep( Repository rep, IMetaStore metaStore, ObjectId id_step, List<DatabaseMeta> databases )
throws KettleException {
try {
awsAccessKey = Encr.decryptPasswordOptionallyEncrypted( rep.getStepAttributeString( id_step, "aws_access_key" ) );
awsSecretKey = Encr.decryptPasswordOptionallyEncrypted( rep.getStepAttributeString( id_step, "aws_secret_key" ) );
bucket = rep.getStepAttributeString( id_step, "bucket" );
filename = rep.getStepAttributeString( id_step, "filename" );
filenameField = rep.getStepAttributeString( id_step, "filename_field" );
Expand All @@ -265,12 +239,6 @@ public void readRep( Repository rep, IMetaStore metaStore, ObjectId id_step, Lis
lazyConversionActive = rep.getStepAttributeBoolean( id_step, "lazy_conversion" );
runningInParallel = rep.getStepAttributeBoolean( id_step, "parallel" );
String awsDefaultCredentials = rep.getStepAttributeString( id_step, "use_aws_default_credentials" );
if ( awsDefaultCredentials == null || awsDefaultCredentials.equalsIgnoreCase( "N" ) ) {
this.useAwsDefaultCredentials = false;
} else {
this.useAwsDefaultCredentials = true;
}

int nrfields = rep.countNrStepAttributes( id_step, "field_name" );

allocate( nrfields );
Expand Down Expand Up @@ -298,10 +266,6 @@ public void readRep( Repository rep, IMetaStore metaStore, ObjectId id_step, Lis
public void saveRep( Repository rep, IMetaStore metaStore, ObjectId id_transformation, ObjectId id_step )
throws KettleException {
try {
rep.saveStepAttribute( id_transformation, id_step, "aws_secret_key", Encr.encryptPasswordIfNotUsingVariables(
awsSecretKey ) );
rep.saveStepAttribute( id_transformation, id_step, "aws_access_key", Encr.encryptPasswordIfNotUsingVariables(
awsAccessKey ) );
rep.saveStepAttribute( id_transformation, id_step, "bucket", bucket );
rep.saveStepAttribute( id_transformation, id_step, "filename", filename );
rep.saveStepAttribute( id_transformation, id_step, "filename_field", filenameField );
Expand All @@ -313,7 +277,6 @@ public void saveRep( Repository rep, IMetaStore metaStore, ObjectId id_transform
rep.saveStepAttribute( id_transformation, id_step, "header", headerPresent );
rep.saveStepAttribute( id_transformation, id_step, "lazy_conversion", lazyConversionActive );
rep.saveStepAttribute( id_transformation, id_step, "parallel", runningInParallel );
rep.saveStepAttribute( id_transformation, id_step, "use_aws_default_credentials", useAwsDefaultCredentials );

for ( int i = 0; i < inputFields.length; i++ ) {
TextFileInputField field = inputFields[i];
Expand Down Expand Up @@ -695,21 +658,6 @@ public void setRunningInParallel( boolean runningInParallel ) {
this.runningInParallel = runningInParallel;
}

/**
* @return the useAwsDefaultCredentials
*/
public boolean getUseAwsDefaultCredentials() {
return this.useAwsDefaultCredentials;
}

/**
* @param useAwsDefaultCredentials
* the useAwsDefaultCredentials to set
*/
public void setUseAwsDefaultCredentials( boolean useAwsDefaultCredentials ) {
this.useAwsDefaultCredentials = useAwsDefaultCredentials;
}

/**
* @return the bucket
*/
Expand All @@ -725,48 +673,10 @@ public void setBucket( String bucket ) {
this.bucket = bucket;
}

/**
* @return the awsAccessKey
*/
public String getAwsAccessKey() {
return awsAccessKey;
}

/**
* @param awsAccessKey
* the awsAccessKey to set
*/
public void setAwsAccessKey( String awsAccessKey ) {
this.awsAccessKey = awsAccessKey;
}

/**
* @return the awsSecretKey
*/
public String getAwsSecretKey() {
return awsSecretKey;
}

/**
* @param awsSecretKey
* the awsSecretKey to set
*/
public void setAwsSecretKey( String awsSecretKey ) {
this.awsSecretKey = awsSecretKey;
}

public S3Service getS3Service( VariableSpace space ) throws S3ServiceException {
String accessKey = Encr.decryptPasswordOptionallyEncrypted( space.environmentSubstitute( awsAccessKey ) );
String secretKey = Encr.decryptPasswordOptionallyEncrypted( space.environmentSubstitute( awsSecretKey ) );
AWSCredentials credentials = null;

if ( getUseAwsDefaultCredentials() ) {
com.amazonaws.auth.AWSCredentials defaultCredentials = DefaultAWSCredentialsProviderChain.getInstance().getCredentials();
credentials = new AWSCredentials( defaultCredentials.getAWSAccessKeyId(), defaultCredentials.getAWSSecretKey() );
} else {
credentials = new AWSCredentials( accessKey, secretKey );
}

com.amazonaws.auth.AWSCredentials defaultCredentials = DefaultAWSCredentialsProviderChain.getInstance().getCredentials();
AWSCredentials credentials = new AWSCredentials( defaultCredentials.getAWSAccessKeyId(), defaultCredentials.getAWSSecretKey() );
S3Service s3service = new RestS3Service( credentials );
return s3service;
}
Expand Down
15 changes: 0 additions & 15 deletions ...e/src/test/java/org/pentaho/di/trans/steps/s3csvinput/S3CsvInputMetaNewInjectionTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,21 +38,6 @@ public void setup() {

@Test
public void test() throws Exception {
check( "AWS_ACCESS_KEY", new StringGetter() {
public String get() {
return meta.getAwsAccessKey();
}
} );
check( "AWS_SECRET_KEY", new StringGetter() {
public String get() {
return meta.getAwsSecretKey();
}
} );
check( "USE_AWS_DEFAULT_CREDENTIALS", new BooleanGetter() {
public boolean get() {
return meta.getUseAwsDefaultCredentials();
}
} );
check( "BUCKET", new StringGetter() {
public String get() {
return meta.getBucket();
Expand Down
42 changes: 2 additions & 40 deletions ...csvinput/core/src/test/java/org/pentaho/di/trans/steps/s3csvinput/S3CsvInputMetaTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
*
* Pentaho Data Integration
*
* Copyright (C) 2002-2017 by Hitachi Vantara : http://www.pentaho.com
* Copyright (C) 2002-2018 by Hitachi Vantara : http://www.pentaho.com
*
*******************************************************************************
*
Expand All @@ -21,17 +21,12 @@
******************************************************************************/
package org.pentaho.di.trans.steps.s3csvinput;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.fail;

import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.jets3t.service.S3Service;
import org.jets3t.service.S3ServiceException;
import org.junit.BeforeClass;
import org.junit.Test;
import org.pentaho.di.core.Const;
Expand All @@ -40,7 +35,6 @@
import org.pentaho.di.core.exception.KettleException;
import org.pentaho.di.core.plugins.PluginRegistry;
import org.pentaho.di.core.util.EnvUtil;
import org.pentaho.di.core.variables.Variables;
import org.pentaho.di.trans.steps.loadsave.LoadSaveTester;
import org.pentaho.di.trans.steps.loadsave.validator.ArrayLoadSaveValidator;
import org.pentaho.di.trans.steps.loadsave.validator.FieldLoadSaveValidator;
Expand Down Expand Up @@ -70,7 +64,7 @@ public static void setUp() throws KettleException {

@Test
public void testSerialization() throws KettleException {
List<String> attributes = Arrays.asList( "AwsAccessKey", "AwsSecretKey", "Bucket", "Filename", "FilenameField",
List<String> attributes = Arrays.asList( "Bucket", "Filename", "FilenameField",
"RowNumField", "IncludingFilename", "Delimiter", "Enclosure", "HeaderPresent", "MaxLineSize",
"LazyConversionActive", "RunningInParallel", "InputFields" );

Expand All @@ -84,36 +78,4 @@ public void testSerialization() throws KettleException {
tester.testSerialization();
}

@Test
public void testGetS3Service_notEncryptedKeys() {
S3CsvInputMeta s3CvsInput = new S3CsvInputMeta();
s3CvsInput.setAwsAccessKey( TEST_ACCESS_KEY );
s3CvsInput.setAwsSecretKey( TEST_AWS_SECRET_KEY );

try {
S3Service s3Service = s3CvsInput.getS3Service( new Variables() );
assertNotNull( s3Service );
assertEquals( TEST_ACCESS_KEY, s3Service.getProviderCredentials().getAccessKey() );
assertEquals( TEST_AWS_SECRET_KEY, s3Service.getProviderCredentials().getSecretKey() );
} catch ( S3ServiceException e ) {
fail( "No exception should be thrown. But it was:" + e.getLocalizedMessage() );
}
}

@Test
public void testGetS3Service_WithEncryptedKeys() {
S3CsvInputMeta s3CvsInput = new S3CsvInputMeta();
s3CvsInput.setAwsAccessKey( TEST_ACCESS_KEY_ENCRYPTED );
s3CvsInput.setAwsSecretKey( TEST_AWS_SECRET_KEY_ENCRYPTED );

try {
S3Service s3Service = s3CvsInput.getS3Service( new Variables() );
assertNotNull( s3Service );
assertEquals( TEST_ACCESS_KEY, s3Service.getProviderCredentials().getAccessKey() );
assertEquals( TEST_AWS_SECRET_KEY, s3Service.getProviderCredentials().getSecretKey() );
} catch ( S3ServiceException e ) {
fail( "No exception should be thrown. But it was:" + e.getLocalizedMessage() );
}
}

}
Loading

0 comments on commit 656fbb7

Please sign in to comment.