forked from pentaho/pentaho-kettle
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[PDI-18440][PDI-18442] Fixes issues with null v.s. empty values (pent…
…aho#7232) * [PDI-18440][PDI-18442] Adds new Kettle properties * [PDI-18440] Causes the 'Get XML Data' step to yield null values on missing elements * [PDI-18442] Allows the 'Filter rows' step to filter empty values * [PDI-18440] The 'Data grid' should yield null values for empty values when the metadata doesn't specify 'Set empty string'
- Loading branch information
1 parent
92f55c1
commit 6d0e66d
Showing
8 changed files
with
308 additions
and
28 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
79 changes: 79 additions & 0 deletions
79
core/src/test/java/org/pentaho/di/core/row/value/ValueMetaBaseTest_NullEmpty.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,79 @@ | ||
| /*! ****************************************************************************** | ||
| * | ||
| * Pentaho Data Integration | ||
| * | ||
| * Copyright (C) 2020 by Hitachi Vantara : http://www.pentaho.com | ||
| * | ||
| ******************************************************************************* | ||
| * | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); | ||
| * you may not use this file except in compliance with | ||
| * the License. You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| * | ||
| ******************************************************************************/ | ||
|
|
||
| package org.pentaho.di.core.row.value; | ||
|
|
||
| import org.junit.Test; | ||
| import org.pentaho.di.core.Const; | ||
|
|
||
| import static org.junit.Assert.assertEquals; | ||
| import static org.junit.Assert.assertNull; | ||
|
|
||
| public class ValueMetaBaseTest_NullEmpty { | ||
|
|
||
| /** | ||
| * By default, converting null value to a string value will yield a null value. | ||
| * This is the expected behavior in current and past versions. | ||
| */ | ||
| @Test | ||
| public void convertDataFromStringWithDefaults() throws Exception { | ||
| System.setProperty( Const.KETTLE_EMPTY_STRING_DIFFERS_FROM_NULL, "N" ); | ||
| System.setProperty( Const.KETTLE_DO_NOT_NORMALIZE_NULL_STRING_TO_EMPTY, "N" ); | ||
|
|
||
| ValueMetaBase out = new ValueMetaString(); | ||
| ValueMetaBase value = new ValueMetaString(); | ||
|
|
||
| Object data = out.convertDataFromString( null, value, null, null, 0 ); | ||
| assertNull( data ); | ||
| } | ||
|
|
||
| /** | ||
| * When KETTLE_EMPTY_STRING_DIFFERS_FROM_NULL is set to "Y" whe start getting unexpected results, see PDI-18440. | ||
| * This flag should have no effect in data conversions, only when comparing values. | ||
| */ | ||
| @Test | ||
| public void convertDataFromStringWithEmptyDiffersFromNull() throws Exception { | ||
| System.setProperty( Const.KETTLE_EMPTY_STRING_DIFFERS_FROM_NULL, "Y" ); | ||
| System.setProperty( Const.KETTLE_DO_NOT_NORMALIZE_NULL_STRING_TO_EMPTY, "N" ); | ||
|
|
||
| ValueMetaBase out = new ValueMetaString(); | ||
| ValueMetaBase value = new ValueMetaString(); | ||
|
|
||
| Object data = out.convertDataFromString( null, value, null, null, 0 ); | ||
| assertEquals( "", data ); | ||
| } | ||
|
|
||
| /** | ||
| * The new KETTLE_DO_NOT_NORMALIZE_NULL_STRING_TO_EMPTY flag fixes PDI-18440 resetting the behavior to what is expected. | ||
| */ | ||
| @Test | ||
| public void convertDataFromStringWithEmptyDiffersFromNullAndDoNotNormalize() throws Exception { | ||
| System.setProperty( Const.KETTLE_EMPTY_STRING_DIFFERS_FROM_NULL, "Y" ); | ||
| System.setProperty( Const.KETTLE_DO_NOT_NORMALIZE_NULL_STRING_TO_EMPTY, "Y" ); | ||
|
|
||
| ValueMetaBase out = new ValueMetaString(); | ||
| ValueMetaBase value = new ValueMetaString(); | ||
|
|
||
| Object data = out.convertDataFromString( null, value, null, null, 0 ); | ||
| assertNull( data ); | ||
| } | ||
| } |
81 changes: 81 additions & 0 deletions
81
core/src/test/java/org/pentaho/di/core/xml/XMLHandlerTest.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,81 @@ | ||
| /*! ****************************************************************************** | ||
| * | ||
| * Pentaho Data Integration | ||
| * | ||
| * Copyright (C) 2020 by Hitachi Vantara : http://www.pentaho.com | ||
| * | ||
| ******************************************************************************* | ||
| * | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); | ||
| * you may not use this file except in compliance with | ||
| * the License. You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| * | ||
| ******************************************************************************/ | ||
|
|
||
| package org.pentaho.di.core.xml; | ||
|
|
||
| import org.junit.Test; | ||
| import org.pentaho.di.core.Const; | ||
| import org.w3c.dom.Element; | ||
| import org.w3c.dom.Node; | ||
| import org.w3c.dom.NodeList; | ||
|
|
||
| import static org.junit.Assert.assertEquals; | ||
| import static org.junit.Assert.assertNull; | ||
| import static org.mockito.Mockito.doReturn; | ||
| import static org.mockito.Mockito.mock; | ||
|
|
||
| public class XMLHandlerTest { | ||
|
|
||
| @Test | ||
| public void getTagValueWithNullNode() { | ||
| assertNull( XMLHandler.getTagValue( null, "text" ) ); | ||
| } | ||
|
|
||
| /** | ||
| * Default behavior, an empty XML tag in the "Filter rows" step meta will be considered {@code null}. | ||
| * This will prevent filtering rows with empty values. | ||
| */ | ||
| @Test | ||
| public void getTagValueEmptyTagYieldsNullValue() { | ||
| System.setProperty( Const.KETTLE_XML_EMPTY_TAG_YIELDS_EMPTY_VALUE, "N" ); | ||
| assertNull( XMLHandler.getTagValue( getNode(), "text" ) ); | ||
| } | ||
|
|
||
| /** | ||
| * An empty XML tag in the "Filter rows" step meta will be considered an empty string. | ||
| * This will allow filtering rows with empty values. | ||
| */ | ||
| @Test | ||
| public void getTagValueEmptyTagYieldsEmptyValue() { | ||
| System.setProperty( Const.KETTLE_XML_EMPTY_TAG_YIELDS_EMPTY_VALUE, "Y" ); | ||
| assertEquals( "", XMLHandler.getTagValue( getNode(), "text" ) ); | ||
| } | ||
|
|
||
| private Node getNode() { | ||
| Element first = mock( Element.class ); | ||
| doReturn( null ).when( first ).getNodeValue(); | ||
|
|
||
| Node child = mock( Node.class ); | ||
| doReturn( "text" ).when( child ).getNodeName(); | ||
| doReturn( first ).when( child ).getFirstChild(); | ||
| doReturn( "" ).when( child ).getTextContent(); | ||
|
|
||
| NodeList children = mock( NodeList.class ); | ||
| doReturn( 1 ).when( children ).getLength(); | ||
| doReturn( child ).when( children ).item( 0 ); | ||
|
|
||
| Node node = mock( Node.class ); | ||
| doReturn( children ).when( node ).getChildNodes(); | ||
|
|
||
| return node; | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.