forked from apache/hive
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
HIVE-14170: Beeline IncrementalRows should buffer rows and incrementa…
…lly re-calculate width if TableOutputFormat is used (Sahil Takiar, reviewed by Tao Li)
- Loading branch information
1 parent
60ec753
commit ebad27d
Showing
9 changed files
with
228 additions
and
12 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
86 changes: 86 additions & 0 deletions
86
beeline/src/java/org/apache/hive/beeline/IncrementalRowsWithNormalization.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,86 @@ | ||
| /** | ||
| * Licensed to the Apache Software Foundation (ASF) under one | ||
| * or more contributor license agreements. See the NOTICE file | ||
| * distributed with this work for additional information | ||
| * regarding copyright ownership. The ASF licenses this file | ||
| * to you under the Apache License, Version 2.0 (the | ||
| * "License"); you may not use this file except in compliance | ||
| * with the License. You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
|
|
||
| /* | ||
| * This source file is based on code taken from SQLLine 1.0.2 | ||
| * See SQLLine notice in LICENSE | ||
| */ | ||
| package org.apache.hive.beeline; | ||
|
|
||
| import java.sql.ResultSet; | ||
| import java.sql.SQLException; | ||
| import java.util.NoSuchElementException; | ||
|
|
||
| import com.google.common.base.Optional; | ||
|
|
||
|
|
||
| /** | ||
| * Extension of {@link IncrementalRows} which buffers "x" number of rows in memory at a time. It | ||
| * uses the {@link BufferedRows} class to do its buffering. The value of "x" is determined by the | ||
| * Beeline option <code>--incrementalBufferRows</code>, which defaults to | ||
| * {@link BeeLineOpts#DEFAULT_INCREMENTAL_BUFFER_ROWS}. Once the initial set of rows are buffered, it | ||
| * will allow the {@link #next()} method to drain the buffer. Once the buffer is empty the next | ||
| * buffer will be fetched until the {@link ResultSet} is empty. The width of the rows are normalized | ||
| * within each buffer using the {@link BufferedRows#normalizeWidths()} method. | ||
| */ | ||
| public class IncrementalRowsWithNormalization extends IncrementalRows { | ||
|
|
||
| private final int incrementalBufferRows; | ||
| private BufferedRows buffer; | ||
|
|
||
| IncrementalRowsWithNormalization(BeeLine beeLine, ResultSet rs) throws SQLException { | ||
| super(beeLine, rs); | ||
|
|
||
| this.incrementalBufferRows = beeLine.getOpts().getIncrementalBufferRows(); | ||
| this.buffer = new BufferedRows(beeLine, rs, Optional.of(this.incrementalBufferRows)); | ||
| this.buffer.normalizeWidths(); | ||
| } | ||
|
|
||
| @Override | ||
| public boolean hasNext() { | ||
| try { | ||
| if (this.buffer.hasNext()) { | ||
| return true; | ||
| } else { | ||
| this.buffer = new BufferedRows(this.beeLine, this.rs, | ||
| Optional.of(this.incrementalBufferRows)); | ||
| if (this.normalizingWidths) { | ||
| this.buffer.normalizeWidths(); | ||
| } | ||
|
|
||
| // Drain the first Row, which just contains column names | ||
| if (!this.buffer.hasNext()) { | ||
| return false; | ||
| } | ||
| this.buffer.next(); | ||
|
|
||
| return this.buffer.hasNext(); | ||
| } | ||
| } catch (SQLException ex) { | ||
| throw new RuntimeException(ex.toString()); | ||
| } | ||
| } | ||
|
|
||
| @Override | ||
| public Object next() { | ||
| if (!hasNext()) { | ||
| throw new NoSuchElementException(); | ||
| } | ||
| return this.buffer.next(); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
90 changes: 90 additions & 0 deletions
90
beeline/src/test/org/apache/hive/beeline/TestIncrementalRowsWithNormalization.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,90 @@ | ||
| /** | ||
| * Licensed to the Apache Software Foundation (ASF) under one | ||
| * or more contributor license agreements. See the NOTICE file | ||
| * distributed with this work for additional information | ||
| * regarding copyright ownership. The ASF licenses this file | ||
| * to you under the Apache License, Version 2.0 (the | ||
| * "License"); you may not use this file except in compliance | ||
| * with the License. You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
|
|
||
| package org.apache.hive.beeline; | ||
|
|
||
| import static org.mockito.Mockito.mock; | ||
| import static org.mockito.Mockito.times; | ||
| import static org.mockito.Mockito.verify; | ||
| import static org.mockito.Mockito.when; | ||
|
|
||
| import java.sql.ResultSet; | ||
| import java.sql.ResultSetMetaData; | ||
| import java.sql.SQLException; | ||
|
|
||
| import org.junit.Test; | ||
|
|
||
| import org.mockito.invocation.InvocationOnMock; | ||
| import org.mockito.stubbing.Answer; | ||
|
|
||
|
|
||
| public class TestIncrementalRowsWithNormalization { | ||
|
|
||
| @Test | ||
| public void testIncrementalRows() throws SQLException { | ||
| Integer incrementalBufferRows = 5; | ||
|
|
||
| // Mock BeeLineOpts | ||
| BeeLineOpts mockBeeLineOpts = mock(BeeLineOpts.class); | ||
| when(mockBeeLineOpts.getIncrementalBufferRows()).thenReturn(incrementalBufferRows); | ||
| when(mockBeeLineOpts.getMaxColumnWidth()).thenReturn(BeeLineOpts.DEFAULT_MAX_COLUMN_WIDTH); | ||
| when(mockBeeLineOpts.getNumberFormat()).thenReturn("default"); | ||
| when(mockBeeLineOpts.getNullString()).thenReturn("NULL"); | ||
|
|
||
| // Mock BeeLine | ||
| BeeLine mockBeeline = mock(BeeLine.class); | ||
| when(mockBeeline.getOpts()).thenReturn(mockBeeLineOpts); | ||
|
|
||
| // MockResultSet | ||
| ResultSet mockResultSet = mock(ResultSet.class); | ||
|
|
||
| ResultSetMetaData mockResultSetMetaData = mock(ResultSetMetaData.class); | ||
| when(mockResultSetMetaData.getColumnCount()).thenReturn(1); | ||
| when(mockResultSetMetaData.getColumnLabel(1)).thenReturn("Mock Table"); | ||
| when(mockResultSet.getMetaData()).thenReturn(mockResultSetMetaData); | ||
|
|
||
| // First 10 calls to resultSet.next() should return true | ||
| when(mockResultSet.next()).thenAnswer(new Answer<Boolean>() { | ||
| private int iterations = 10; | ||
|
|
||
| @Override | ||
| public Boolean answer(InvocationOnMock invocation) { | ||
| return this.iterations-- > 0; | ||
| } | ||
| }); | ||
|
|
||
| when(mockResultSet.getString(1)).thenReturn("Hello World"); | ||
|
|
||
| // IncrementalRows constructor should buffer the first "incrementalBufferRows" rows | ||
| IncrementalRowsWithNormalization incrementalRowsWithNormalization = new IncrementalRowsWithNormalization( | ||
| mockBeeline, mockResultSet); | ||
|
|
||
| // When the first buffer is loaded ResultSet.next() should be called "incrementalBufferRows" times | ||
| verify(mockResultSet, times(5)).next(); | ||
|
|
||
| // Iterating through the buffer should not cause the next buffer to be fetched | ||
| for (int i = 0; i < incrementalBufferRows + 1; i++) { | ||
| incrementalRowsWithNormalization.next(); | ||
| } | ||
| verify(mockResultSet, times(5)).next(); | ||
|
|
||
| // When a new buffer is fetched ResultSet.next() should be called "incrementalBufferRows" more times | ||
| incrementalRowsWithNormalization.next(); | ||
| verify(mockResultSet, times(10)).next(); | ||
| } | ||
| } |