Skip to content

Commit

Permalink
Showing 10 changed files with 267 additions and 279 deletions.
1 change: 0 additions & 1 deletion flink-runtime/pom.xml
Original file line number Diff line number Diff line change
@@ -443,7 +443,6 @@ under the License.
**/runtime/jobmanager/**,
**/runtime/jobmaster/**,
**/runtime/leaderelection/**,
**/runtime/memory/**,
**/runtime/messages/**,
**/runtime/minicluster/**,
**/runtime/operators/**,
Original file line number Diff line number Diff line change
@@ -16,46 +16,45 @@
* limitations under the License.
*/


package org.apache.flink.runtime.memory;

import org.apache.flink.core.memory.DataInputView;
import org.apache.flink.core.memory.MemorySegment;

import java.io.EOFException;
import java.io.IOException;
import java.io.UTFDataFormatException;

import org.apache.flink.core.memory.DataInputView;
import org.apache.flink.core.memory.MemorySegment;


/**
* The base class for all input views that are backed by multiple memory pages. This base class contains all
* decoding methods to read data from a page and detect page boundary crossing. The concrete sub classes must
* implement the methods to provide the next memory page once the boundary is crossed.
*/
public abstract class AbstractPagedInputView implements DataInputView {

private MemorySegment currentSegment;

protected final int headerLength; // the number of bytes to skip at the beginning of each segment

private int positionInSegment; // the offset in the current segment

private int limitInSegment; // the limit in the current segment before switching to the next

private byte[] utfByteBuffer; // reusable byte buffer for utf-8 decoding
private char[] utfCharBuffer; // reusable char buffer for utf-8 decoding


// --------------------------------------------------------------------------------------------
// Constructors
// --------------------------------------------------------------------------------------------

/**
* Creates a new view that starts with the given segment. The input starts directly after the header
* of the given page. If the header size is zero, it starts at the beginning. The specified initial
* limit describes up to which position data may be read from the current segment, before the view must
* advance to the next segment.
*
*
* @param initialSegment The memory segment to start reading from.
* @param initialLimit The position one after the last valid byte in the initial segment.
* @param headerLength The number of bytes to skip at the beginning of each segment for the header. This
@@ -66,14 +65,14 @@ protected AbstractPagedInputView(MemorySegment initialSegment, int initialLimit,
this.positionInSegment = headerLength;
seekInput(initialSegment, headerLength, initialLimit);
}

/**
* Creates a new view that is initially not bound to a memory segment. This constructor is typically
* for views that always seek first.
* <p>
* WARNING: The view is not readable until the first call to either {@link #advance()},
*
* <p>WARNING: The view is not readable until the first call to either {@link #advance()},
* or to {@link #seekInput(MemorySegment, int, int)}.
*
*
* @param headerLength The number of bytes to skip at the beginning of each segment for the header.
*/
protected AbstractPagedInputView(int headerLength) {
@@ -83,73 +82,73 @@ protected AbstractPagedInputView(int headerLength) {
// --------------------------------------------------------------------------------------------
// Page Management
// --------------------------------------------------------------------------------------------

/**
* Gets the memory segment that will be used to read the next bytes from. If the segment is exactly exhausted,
* meaning that the last byte read was the last byte available in the segment, then this segment will
* not serve the next bytes. The segment to serve the next bytes will be obtained through the
* {@link #nextSegment(MemorySegment)} method.
*
*
* @return The current memory segment.
*/
public MemorySegment getCurrentSegment() {
return this.currentSegment;
}

/**
* Gets the position from which the next byte will be read. If that position is equal to the current limit,
* then the next byte will be read from next segment.
*
*
* @return The position from which the next byte will be read.
* @see #getCurrentSegmentLimit()
*/
public int getCurrentPositionInSegment() {
return this.positionInSegment;
}

/**
* Gets the current limit in the memory segment. This value points to the byte one after the last valid byte
* in the memory segment.
*
*
* @return The current limit in the memory segment.
* @see #getCurrentPositionInSegment()
*/
public int getCurrentSegmentLimit() {
return this.limitInSegment;
}

/**
* The method by which concrete subclasses realize page crossing. This method is invoked when the current page
* is exhausted and a new page is required to continue the reading. If no further page is available, this
* method must throw an {@link EOFException}.
*
*
* @param current The current page that was read to its limit. May be {@code null}, if this method is
* invoked for the first time.
* @return The next page from which the reading should continue. May not be {@code null}. If the input is
* exhausted, an {@link EOFException} must be thrown instead.
*
*
* @throws EOFException Thrown, if no further segment is available.
* @throws IOException Thrown, if the method cannot provide the next page due to an I/O related problem.
*/
protected abstract MemorySegment nextSegment(MemorySegment current) throws EOFException, IOException;

/**
* Gets the limit for reading bytes from the given memory segment. This method must return the position
* of the byte after the last valid byte in the given memory segment. When the position returned by this
* method is reached, the view will attempt to switch to the next memory segment.
*
*
* @param segment The segment to determine the limit for.
* @return The limit for the given memory segment.
*/
protected abstract int getLimitForSegment(MemorySegment segment);

/**
* Advances the view to the next memory segment. The reading will continue after the header of the next
* segment. This method uses {@link #nextSegment(MemorySegment)} and {@link #getLimitForSegment(MemorySegment)}
* to get the next segment and set its limit.
*
*
* @throws IOException Thrown, if the next segment could not be obtained.
*
*
* @see #nextSegment(MemorySegment)
* @see #getLimitForSegment(MemorySegment)
*/
@@ -160,11 +159,11 @@ protected final void advance() throws IOException {
this.limitInSegment = getLimitForSegment(this.currentSegment);
this.positionInSegment = this.headerLength;
}

/**
* Sets the internal state of the view such that the next bytes will be read from the given memory segment,
* starting at the given position. The memory segment will provide bytes up to the given limit position.
*
*
* @param segment The segment to read the next bytes from.
* @param positionInSegment The position in the segment to start reading from.
* @param limitInSegment The limit in the segment. When reached, the view will attempt to switch to
@@ -175,7 +174,7 @@ protected void seekInput(MemorySegment segment, int positionInSegment, int limit
this.positionInSegment = positionInSegment;
this.limitInSegment = limitInSegment;
}

/**
* Clears the internal state of the view. After this call, all read attempts will fail, until the
* {@link #advance()} or {@link #seekInput(MemorySegment, int, int)} method have been invoked.
@@ -185,14 +184,14 @@ protected void clear() {
this.positionInSegment = this.headerLength;
this.limitInSegment = headerLength;
}

// --------------------------------------------------------------------------------------------
// Data Input Specific methods
// --------------------------------------------------------------------------------------------

@Override
public int read(byte[] b) throws IOException{
return read(b,0,b.length);
return read(b, 0, b.length);
}

@Override
@@ -220,7 +219,7 @@ public int read(byte[] b, int off, int len) throws IOException{

int bytesRead = 0;
while (true) {
int toRead = Math.min(remaining, len-bytesRead);
int toRead = Math.min(remaining, len - bytesRead);
this.currentSegment.get(this.positionInSegment, b, off, toRead);
off += toRead;
bytesRead += toRead;
@@ -243,17 +242,17 @@ public int read(byte[] b, int off, int len) throws IOException{
return len;
}
}

@Override
public void readFully(byte[] b) throws IOException {
readFully(b, 0, b.length);
}

@Override
public void readFully(byte[] b, int off, int len) throws IOException {
int bytesRead = read(b,off,len);
int bytesRead = read(b, off, len);

if(bytesRead < len){
if (bytesRead < len){
throw new EOFException("There is no enough data left in the DataInputView.");
}
}
@@ -384,7 +383,7 @@ public double readDouble() throws IOException {
@Override
public String readLine() throws IOException {
final StringBuilder bld = new StringBuilder(32);

try {
int b;
while ((b = readUnsignedByte()) != '\n') {
@@ -398,7 +397,7 @@ public String readLine() throws IOException {
if (bld.length() == 0) {
return null;
}

// trim a trailing carriage return
int len = bld.length();
if (len > 0 && bld.charAt(len - 1) == '\r') {
@@ -410,10 +409,10 @@ public String readLine() throws IOException {
@Override
public String readUTF() throws IOException {
final int utflen = readUnsignedShort();

final byte[] bytearr;
final char[] chararr;

if (this.utfByteBuffer == null || this.utfByteBuffer.length < utflen) {
bytearr = new byte[utflen];
this.utfByteBuffer = bytearr;
@@ -429,7 +428,7 @@ public String readUTF() throws IOException {

int c, char2, char3;
int count = 0;
int chararr_count = 0;
int chararrCount = 0;

readFully(bytearr, 0, utflen);

@@ -439,7 +438,7 @@ public String readUTF() throws IOException {
break;
}
count++;
chararr[chararr_count++] = (char) c;
chararr[chararrCount++] = (char) c;
}

while (count < utflen) {
@@ -455,7 +454,7 @@ public String readUTF() throws IOException {
case 7:
/* 0xxxxxxx */
count++;
chararr[chararr_count++] = (char) c;
chararr[chararrCount++] = (char) c;
break;
case 12:
case 13:
@@ -468,7 +467,7 @@ public String readUTF() throws IOException {
if ((char2 & 0xC0) != 0x80) {
throw new UTFDataFormatException("malformed input around byte " + count);
}
chararr[chararr_count++] = (char) (((c & 0x1F) << 6) | (char2 & 0x3F));
chararr[chararrCount++] = (char) (((c & 0x1F) << 6) | (char2 & 0x3F));
break;
case 14:
/* 1110 xxxx 10xx xxxx 10xx xxxx */
@@ -481,23 +480,23 @@ public String readUTF() throws IOException {
if (((char2 & 0xC0) != 0x80) || ((char3 & 0xC0) != 0x80)) {
throw new UTFDataFormatException("malformed input around byte " + (count - 1));
}
chararr[chararr_count++] = (char) (((c & 0x0F) << 12) | ((char2 & 0x3F) << 6) | ((char3 & 0x3F) << 0));
chararr[chararrCount++] = (char) (((c & 0x0F) << 12) | ((char2 & 0x3F) << 6) | ((char3 & 0x3F) << 0));
break;
default:
/* 10xx xxxx, 1111 xxxx */
throw new UTFDataFormatException("malformed input around byte " + count);
}
}
// The number of chars produced may be less than utflen
return new String(chararr, 0, chararr_count);
return new String(chararr, 0, chararrCount);
}

@Override
public int skipBytes(int n) throws IOException {
if (n < 0) {
throw new IllegalArgumentException();
}

int remaining = this.limitInSegment - this.positionInSegment;
if (remaining >= n) {
this.positionInSegment += n;
@@ -512,20 +511,20 @@ public int skipBytes(int n) throws IOException {
}
remaining = this.limitInSegment - this.positionInSegment;
}

int skipped = 0;
while (true) {
int toSkip = Math.min(remaining, n);
n -= toSkip;
skipped += toSkip;

if (n > 0) {
try {
advance();
} catch (EOFException eofex) {
return skipped;
}
remaining = this.limitInSegment - this.positionInSegment;
remaining = this.limitInSegment - this.positionInSegment;
}
else {
this.positionInSegment += toSkip;
@@ -541,7 +540,7 @@ public void skipBytesToRead(int numBytes) throws IOException {
if (numBytes < 0) {
throw new IllegalArgumentException();
}

int remaining = this.limitInSegment - this.positionInSegment;
if (remaining >= numBytes) {
this.positionInSegment += numBytes;
@@ -551,12 +550,12 @@ public void skipBytesToRead(int numBytes) throws IOException {
advance();
remaining = this.limitInSegment - this.positionInSegment;
}

while (true) {
if (numBytes > remaining) {
numBytes -= remaining;
advance();
remaining = this.limitInSegment - this.positionInSegment;
remaining = this.limitInSegment - this.positionInSegment;
}
else {
this.positionInSegment += numBytes;
Loading

0 comments on commit 7c150a6

Please sign in to comment.