Skip to content

Commit

Permalink
[SPARK-26857][SQL] Return UnsafeArrayData for date/timestamp type in …
Browse files Browse the repository at this point in the history
…ColumnarArray.copy()

## What changes were proposed in this pull request?

In apache#23569, the copy method of `ColumnarArray` is implemented.
To further improve it, we can return `UnsafeArrayData` for `date`/`timestamp` type in `ColumnarArray.copy()`.

## How was this patch tested?

Unit test

Closes apache#23761 from gengliangwang/copyDateAndTS.

Authored-by: Gengliang Wang <[email protected]>
Signed-off-by: Takeshi Yamamuro <[email protected]>
  • Loading branch information
gengliangwang authored and maropu committed Feb 13, 2019
1 parent 5a74036 commit 72a349a
Show file tree
Hide file tree
Showing 2 changed files with 30 additions and 2 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -56,9 +56,9 @@ public ArrayData copy() {
return UnsafeArrayData.fromPrimitiveArray(toByteArray());
} else if (dt instanceof ShortType) {
return UnsafeArrayData.fromPrimitiveArray(toShortArray());
} else if (dt instanceof IntegerType) {
} else if (dt instanceof IntegerType || dt instanceof DateType) {
return UnsafeArrayData.fromPrimitiveArray(toIntArray());
} else if (dt instanceof LongType) {
} else if (dt instanceof LongType || dt instanceof TimestampType) {
return UnsafeArrayData.fromPrimitiveArray(toLongArray());
} else if (dt instanceof FloatType) {
return UnsafeArrayData.fromPrimitiveArray(toFloatArray());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,20 @@ class ColumnVectorSuite extends SparkFunSuite with BeforeAndAfterEach {
}
}

testVectors("date", 10, DateType) { testVector =>
(0 until 10).foreach { i =>
testVector.appendInt(i)
}

val array = new ColumnarArray(testVector, 0, 10)
val arrayCopy = array.copy()

(0 until 10).foreach { i =>
assert(array.get(i, DateType) === i)
assert(arrayCopy.get(i, DateType) === i)
}
}

testVectors("long", 10, LongType) { testVector =>
(0 until 10).foreach { i =>
testVector.appendLong(i)
Expand All @@ -122,6 +136,20 @@ class ColumnVectorSuite extends SparkFunSuite with BeforeAndAfterEach {
}
}

testVectors("timestamp", 10, TimestampType) { testVector =>
(0 until 10).foreach { i =>
testVector.appendLong(i)
}

val array = new ColumnarArray(testVector, 0, 10)
val arrayCopy = array.copy()

(0 until 10).foreach { i =>
assert(array.get(i, TimestampType) === i)
assert(arrayCopy.get(i, TimestampType) === i)
}
}

testVectors("float", 10, FloatType) { testVector =>
(0 until 10).foreach { i =>
testVector.appendFloat(i.toFloat)
Expand Down

0 comments on commit 72a349a

Please sign in to comment.