Skip to content

Commit

Permalink
Minor fix in DictionaryCompressionOptimizer
Browse files Browse the repository at this point in the history
If an dictionary column failed to convert to direct encoded due to
stripe size limitation, it is no longer considered as a candidate
to direct conversion. However, we should still update the its
growth info and use the stats when select the dictionary column to
convert.
  • Loading branch information
wenleix committed Sep 4, 2018
1 parent bd1e2a7 commit 9751eb8
Showing 1 changed file with 12 additions and 8 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,9 @@ public void optimize(int bufferedBytes, int stripeRowCount)
.sum();

// update the dictionary growth history
directConversionCandidates.forEach(column -> column.updateHistory(stripeRowCount));
allWriters.stream()
.filter(writer -> !writer.isDirectEncoded())
.forEach(column -> column.updateHistory(stripeRowCount));

if (dictionaryMemoryBytes <= dictionaryMemoryMaxBytesLow) {
return;
Expand Down Expand Up @@ -246,14 +248,16 @@ private DictionaryCompressionProjection selectDictionaryColumnToConvert(int tota
long totalDictionaryBytesPerNewRow = 0;
long totalDictionaryIndexBytesPerRow = 0;

for (DictionaryColumnManager column : directConversionCandidates) {
totalDictionaryRawBytes += column.getRawBytes();
totalDictionaryBytes += column.getDictionaryBytes();
totalDictionaryIndexBytes += column.getIndexBytes();
for (DictionaryColumnManager column : allWriters) {
if (!column.isDirectEncoded()) {
totalDictionaryRawBytes += column.getRawBytes();
totalDictionaryBytes += column.getDictionaryBytes();
totalDictionaryIndexBytes += column.getIndexBytes();

totalDictionaryRawBytesPerRow += column.getRawBytesPerRow();
totalDictionaryBytesPerNewRow += column.getDictionaryBytesPerFutureRow();
totalDictionaryIndexBytesPerRow += column.getIndexBytesPerRow();
totalDictionaryRawBytesPerRow += column.getRawBytesPerRow();
totalDictionaryBytesPerNewRow += column.getDictionaryBytesPerFutureRow();
totalDictionaryIndexBytesPerRow += column.getIndexBytesPerRow();
}
}

long totalUncompressedBytesPerRow = totalNonDictionaryBytesPerRow + totalDictionaryRawBytesPerRow;
Expand Down

0 comments on commit 9751eb8

Please sign in to comment.