Skip to content

Commit d21c1a9

Browse files
arnocandelarnocandel
committed
Merge branch 'arno-chunks'
2 parents 0fa07f2 + ffc7f11 commit d21c1a9

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

42 files changed

+1375
-334
lines changed

‎src/main/java/water/Model.java

+3-3
Original file line numberDiff line numberDiff line change
@@ -753,7 +753,7 @@ public final void scoreCrossValidation(Job.ValidatedJob job, Frame source, Vec r
753753
for (int c=(isClassifier() ? 1 : 0); c<cv_preds[i].numCols(); ++c) {
754754
Vec.Writer vw = cv_pred.vec(c).open();
755755
try {
756-
for (int r=0; r < cv_preds[i].numRows(); ++r) {
756+
for (long r=0; r < cv_preds[i].numRows(); ++r) {
757757
vw.set(offsets[i] + r, cv_preds[i].vec(c).at(r));
758758
}
759759
} finally {
@@ -765,12 +765,12 @@ public final void scoreCrossValidation(Job.ValidatedJob job, Frame source, Vec r
765765
float[] probs = new float[cv_preds[i].numCols()];
766766
Vec.Writer vw = cv_pred.vec(0).open();
767767
try {
768-
for (int r = 0; r < cv_preds[i].numRows(); ++r) {
768+
for (long r = 0; r < cv_preds[i].numRows(); ++r) {
769769
//probs[0] stays 0, is not used in getPrediction
770770
for (int c = 1; c < cv_preds[i].numCols(); ++c) {
771771
probs[c] = (float) cv_preds[i].vec(c).at(r);
772772
}
773-
final int label = ModelUtils.getPrediction(probs, r);
773+
final int label = ModelUtils.getPrediction(probs, (int)r);
774774
vw.set(offsets[i] + r, label);
775775
}
776776
} finally {

‎src/main/java/water/fvec/AppendableVec.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -48,12 +48,12 @@ synchronized void closeChunk( NewChunk chk) {
4848
_espc = Arrays.copyOf(_espc,_espc.length<<1);
4949
_chunkTypes = Arrays.copyOf(_chunkTypes,_chunkTypes.length<<1);
5050
}
51-
_espc[cidx] = chk._len2;
51+
_espc[cidx] = chk._len;
5252
_chunkTypes[cidx] = chk.type();
5353
_naCnt += chk._naCnt;
5454
_strCnt += chk._strCnt;
5555
for( int i=0; i<_timCnt.length; i++ ) _timCnt[i] += chk._timCnt[i];
56-
_totalCnt += chk._len2;
56+
_totalCnt += chk._len;
5757
}
5858

5959
// What kind of data did we find? NA's? Strings-only? Floats or Ints?

‎src/main/java/water/fvec/C0DChunk.java

+4-9
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,6 @@
11
package water.fvec;
22

3-
import java.util.Arrays;
43
import water.AutoBuffer;
5-
import water.MemoryManager;
64
import water.UDP;
75

86
/**
@@ -37,14 +35,11 @@ public class C0DChunk extends Chunk {
3735
}
3836
@Override public int sparseLen(){return _con == 0?0:_len;}
3937
@Override NewChunk inflate_impl(NewChunk nc) {
38+
nc.set_len(nc.set_sparseLen(0)); //Work-around because NewChunk(Chunk) constructor already set the length from the Chunk
4039
if(_con == 0) {
41-
nc._id = new int[0];
42-
nc._ls = new long[0];
43-
nc._xs = new int[0];
44-
}
45-
else {
46-
nc._ds = MemoryManager.malloc8d(_len);
47-
Arrays.fill(nc._ds,_con);
40+
nc.addZeros(len());
41+
} else {
42+
for (int i=0; i<len(); ++i) nc.addNum(_con);
4843
}
4944
return nc;
5045
}

‎src/main/java/water/fvec/C0LChunk.java

+10-10
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@
22

33
import java.util.Arrays;
44
import water.AutoBuffer;
5-
import water.MemoryManager;
65
import water.UDP;
76

87
/**
@@ -33,15 +32,16 @@ public class C0LChunk extends Chunk {
3332
return this;
3433
}
3534
@Override NewChunk inflate_impl(NewChunk nc) {
36-
if(_con == 0) {
37-
nc._id = new int[0];
38-
nc._ls = new long[0];
39-
nc._xs = new int[0];
40-
}
41-
else {
42-
nc._ls = MemoryManager.malloc8(_len);
43-
Arrays.fill(nc._ls,_con);
44-
nc._xs = MemoryManager.malloc4(_len);
35+
if(_con != 0) {
36+
nc.alloc_mantissa(len());
37+
Arrays.fill(nc.mantissa(), _con);
38+
nc.alloc_exponent(len());
39+
nc.set_len(nc.set_sparseLen(len()));
40+
} else {
41+
nc.set_len(nc.set_sparseLen(0));
42+
nc.set_sparse(0);
43+
nc.addZeros(len());
44+
assert(nc.sparseLen() == 0);
4545
}
4646
return nc;
4747
}

‎src/main/java/water/fvec/C16Chunk.java

+3-4
Original file line numberDiff line numberDiff line change
@@ -39,10 +39,9 @@ public class C16Chunk extends Chunk {
3939
}
4040
@Override NewChunk inflate_impl(NewChunk nc) {
4141
//nothing to inflate - just copy
42-
nc._ls = MemoryManager.malloc8 (_len);
43-
nc._ds = MemoryManager.malloc8d(_len);
44-
nc._len = _len;
45-
nc._len2 = _len;
42+
nc.alloc_mantissa(len());
43+
nc.alloc_doubles(len());
44+
nc.set_len(nc.set_sparseLen(len()));
4645
for( int i=0; i<_len; i++ ) { //use unsafe?
4746
nc._ls[i] = UDP.get8(_mem,(i<<4) );
4847
nc._ds[i] = Double.longBitsToDouble(UDP.get8(_mem,(i<<4)+8));

‎src/main/java/water/fvec/C1Chunk.java

+5-5
Original file line numberDiff line numberDiff line change
@@ -37,12 +37,12 @@ public class C1Chunk extends Chunk {
3737
return this;
3838
}
3939
@Override NewChunk inflate_impl(NewChunk nc) {
40-
nc._xs = MemoryManager.malloc4(_len);
41-
nc._ls = MemoryManager.malloc8(_len);
42-
for( int i=0; i<_len; i++ ) {
40+
nc.set_len(nc.set_sparseLen(0));
41+
final int len = len();
42+
for( int i=0; i<len; i++ ) {
4343
int res = 0xFF&_mem[i+OFF];
44-
if( res == C1Chunk._NA ) nc.setNA_impl2(i);
45-
else nc._ls[i] = res;
44+
if( res == _NA ) nc.addNA();
45+
else nc.addNum(res,0);
4646
}
4747
return nc;
4848
}

‎src/main/java/water/fvec/C1NChunk.java

+5-4
Original file line numberDiff line numberDiff line change
@@ -25,10 +25,11 @@ public class C1NChunk extends Chunk {
2525
return this;
2626
}
2727
@Override NewChunk inflate_impl(NewChunk nc) {
28-
nc._xs = MemoryManager.malloc4(_len);
29-
nc._ls = MemoryManager.malloc8(_len);
30-
for( int i=0; i<_len; i++ )
31-
nc._ls[i] = 0xFF&_mem[i+OFF];
28+
nc.alloc_exponent(len());
29+
nc.alloc_mantissa(len());
30+
for( int i=0; i< len(); i++ )
31+
nc.mantissa()[i] = 0xFF&_mem[i+OFF];
32+
nc.set_len(nc.set_sparseLen(len()));
3233
return nc;
3334
}
3435
}

‎src/main/java/water/fvec/C1SChunk.java

+7-6
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
import java.util.Arrays;
44
import water.*;
55
import water.parser.DParseTask;
6+
import static water.parser.DParseTask.fitsIntoInt;
67

78
/**
89
* The scale/bias function, where data is in SIGNED bytes before scaling.
@@ -49,13 +50,13 @@ public class C1SChunk extends Chunk {
4950
}
5051
@Override NewChunk inflate_impl(NewChunk nc) {
5152
double dx = Math.log10(_scale);
52-
assert DParseTask.fitsIntoInt(dx);
53-
Arrays.fill(nc._xs = MemoryManager.malloc4(_len), (int)dx);
54-
nc._ls = MemoryManager.malloc8(_len);
55-
for( int i=0; i<_len; i++ ) {
53+
assert fitsIntoInt(dx);
54+
nc.set_len(nc.set_sparseLen(0));
55+
final int len = len();
56+
for( int i=0; i<len; i++ ) {
5657
int res = 0xFF&_mem[i+OFF];
57-
if( res == C1Chunk._NA ) nc.setNA_impl2(i);
58-
else nc._ls[i] = res+_bias;
58+
if( res == C1Chunk._NA ) nc.addNA();
59+
else nc.addNum((long)(res+_bias),(int)dx);
5960
}
6061
return nc;
6162
}

‎src/main/java/water/fvec/C2Chunk.java

+5-5
Original file line numberDiff line numberDiff line change
@@ -41,12 +41,12 @@ public class C2Chunk extends Chunk {
4141
return this;
4242
}
4343
@Override NewChunk inflate_impl(NewChunk nc) {
44-
nc._xs = MemoryManager.malloc4(_len);
45-
nc._ls = MemoryManager.malloc8(_len);
46-
for( int i=0; i<_len; i++ ) {
44+
nc.set_len(nc.set_sparseLen(0));
45+
final int len = len();
46+
for( int i=0; i<len; i++ ) {
4747
int res = UDP.get2(_mem,(i<<1)+OFF);
48-
if( res == C2Chunk._NA ) nc.setNA_impl2(i);
49-
else nc._ls[i] = res;
48+
if( res == _NA ) nc.addNA();
49+
else nc.addNum(res,0);
5050
}
5151
return nc;
5252
}

‎src/main/java/water/fvec/C2SChunk.java

+7-6
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
import java.util.Arrays;
44
import water.*;
55
import water.parser.DParseTask;
6+
import static water.parser.DParseTask.fitsIntoInt;
67

78
/**
89
* The scale/bias function, where data is in SIGNED bytes before scaling.
@@ -56,13 +57,13 @@ public class C2SChunk extends Chunk {
5657
}
5758
@Override NewChunk inflate_impl(NewChunk nc) {
5859
double dx = Math.log10(_scale);
59-
assert DParseTask.fitsIntoInt(dx);
60-
Arrays.fill(nc._xs = MemoryManager.malloc4(_len), (int)dx);
61-
nc._ls = MemoryManager.malloc8(_len);
62-
for( int i=0; i<_len; i++ ) {
60+
assert fitsIntoInt(dx);
61+
nc.set_len(nc.set_sparseLen(0));
62+
final int len = len();
63+
for( int i=0; i<len; i++ ) {
6364
int res = UDP.get2(_mem,(i<<1)+OFF);
64-
if( res == C2Chunk._NA ) nc.setNA_impl2(i);
65-
else nc._ls[i] = res+_bias;
65+
if( res == C2Chunk._NA ) nc.addNA();
66+
else nc.addNum((long)(res+_bias),(int)dx);
6667
}
6768
return nc;
6869
}

‎src/main/java/water/fvec/C4Chunk.java

+7-8
Original file line numberDiff line numberDiff line change
@@ -36,14 +36,13 @@ public class C4Chunk extends Chunk {
3636
return this;
3737
}
3838
@Override NewChunk inflate_impl(NewChunk nc) {
39-
//nothing to inflate - just copy
40-
nc._ls = MemoryManager.malloc8(_len);
41-
nc._xs = MemoryManager.malloc4(_len);
42-
nc._len = 0;
43-
nc._len2 = 0;
44-
for( int i=0; i<_len; i++ ) //use unsafe?
45-
if(isNA0(i))nc.addNA();
46-
else nc.addNum(at80(i),0);
39+
nc.set_len(nc.set_sparseLen(0));
40+
final int len = len();
41+
for( int i=0; i<len; i++ ) {
42+
int res = UDP.get4(_mem,(i<<2));
43+
if( res == _NA ) nc.addNA();
44+
else nc.addNum(res,0);
45+
}
4746
return nc;
4847
}
4948
}

‎src/main/java/water/fvec/C4FChunk.java

+7-4
Original file line numberDiff line numberDiff line change
@@ -34,10 +34,13 @@ public class C4FChunk extends Chunk {
3434
return this;
3535
}
3636
@Override NewChunk inflate_impl(NewChunk nc) {
37-
//nothing to inflate - just copy
38-
nc._ds = MemoryManager.malloc8d(_len);
39-
for( int i=0; i<_len; i++ ) //use unsafe?
40-
nc._ds[i] = UDP.get4f(_mem, (i << 2));
37+
nc.set_len(nc.set_sparseLen(0));
38+
final int len = len();
39+
for( int i=0; i<len; i++ ) {
40+
float res = UDP.get4f(_mem,(i<<2));
41+
if( Float.isNaN(res) ) nc.addNum(Double.NaN);
42+
else nc.addNum(res);
43+
}
4144
return nc;
4245
}
4346
// 3.3333333e33

‎src/main/java/water/fvec/C4SChunk.java

+12-11
Original file line numberDiff line numberDiff line change
@@ -3,19 +3,20 @@
33
import java.util.Arrays;
44
import water.*;
55
import water.parser.DParseTask;
6+
import static water.parser.DParseTask.fitsIntoInt;
67

78
/**
89
* The scale/bias function, where data is in SIGNED bytes before scaling.
910
*/
1011
public class C4SChunk extends Chunk {
1112
static private final long _NA = Integer.MIN_VALUE;
12-
static final int OFF=8+4;
13+
static final int OFF=8+8;
1314
public double _scale;
14-
int _bias;
15-
C4SChunk( byte[] bs, int bias, double scale ) { _mem=bs; _start = -1; _len = (_mem.length-OFF)>>2;
15+
long _bias;
16+
C4SChunk( byte[] bs, long bias, double scale ) { _mem=bs; _start = -1; _len = (_mem.length-OFF)>>2;
1617
_bias = bias; _scale = scale;
1718
UDP.set8d(_mem,0,scale);
18-
UDP.set4 (_mem,8,bias );
19+
UDP.set8(_mem, 8, bias);
1920
}
2021
@Override protected final long at8_impl( int i ) {
2122
long res = UDP.get4(_mem,(i<<2)+OFF);
@@ -45,18 +46,18 @@ public class C4SChunk extends Chunk {
4546
_start = -1;
4647
_len = (_mem.length-OFF)>>2;
4748
_scale= UDP.get8d(_mem,0);
48-
_bias = UDP.get4 (_mem,8);
49+
_bias = UDP.get8(_mem, 8);
4950
return this;
5051
}
5152
@Override NewChunk inflate_impl(NewChunk nc) {
5253
double dx = Math.log10(_scale);
53-
assert DParseTask.fitsIntoInt(dx);
54-
Arrays.fill(nc._xs = MemoryManager.malloc4(_len), (int)dx);
55-
nc._ls = MemoryManager.malloc8(_len);
56-
for( int i=0; i<_len; i++ ) {
54+
assert fitsIntoInt(dx);
55+
nc.set_len(nc.set_sparseLen(0));
56+
final int len = len();
57+
for( int i=0; i<len; i++ ) {
5758
int res = UDP.get4(_mem,(i<<2)+OFF);
58-
if( res == _NA ) nc.setNA_impl2(i);
59-
else nc._ls[i] = res+_bias;
59+
if( res == _NA ) nc.addNA();
60+
else nc.addNum(res+_bias,(int)dx);
6061
}
6162
return nc;
6263
}

‎src/main/java/water/fvec/C8Chunk.java

+2-6
Original file line numberDiff line numberDiff line change
@@ -32,12 +32,8 @@ public class C8Chunk extends Chunk {
3232
return this;
3333
}
3434
@Override NewChunk inflate_impl(NewChunk nc) {
35-
//nothing to inflate - just copy
36-
nc._ls = MemoryManager.malloc8(_len);
37-
nc._xs = MemoryManager.malloc4(_len);
38-
nc._len = 0;
39-
nc._len2 = 0;
40-
for( int i=0; i<_len; i++ ) //use unsafe?
35+
nc.set_len(nc.set_sparseLen(0));
36+
for( int i=0; i< len(); i++ )
4137
if(isNA0(i))nc.addNA();
4238
else nc.addNum(at80(i),0);
4339
return nc;

‎src/main/java/water/fvec/C8DChunk.java

+4-4
Original file line numberDiff line numberDiff line change
@@ -36,10 +36,10 @@ public class C8DChunk extends Chunk {
3636
return this;
3737
}
3838
@Override NewChunk inflate_impl(NewChunk nc) {
39-
//nothing to inflate - just copy
40-
nc._ds = MemoryManager.malloc8d(_len);
41-
for( int i=0; i<_len; i++ ) //use unsafe?
42-
nc._ds[i] = UDP.get8d(_mem,(i<<3));
39+
nc.alloc_doubles(len());
40+
for( int i=0; i< len(); i++ )
41+
nc.doubles()[i] = UDP.get8d(_mem,(i<<3));
42+
nc.set_len(nc.set_sparseLen(len()));
4343
return nc;
4444
}
4545
// 3.3333333e33

‎src/main/java/water/fvec/CBSChunk.java

+4-5
Original file line numberDiff line numberDiff line change
@@ -58,12 +58,11 @@ protected byte atb(int idx) {
5858
return this;
5959
}
6060
@Override NewChunk inflate_impl(NewChunk nc) {
61-
nc._xs = MemoryManager.malloc4(_len);
62-
nc._ls = MemoryManager.malloc8(_len);
63-
for (int i=0; i<_len; i++) {
61+
nc.set_len(nc.set_sparseLen(0));
62+
for (int i=0; i< len(); i++) {
6463
int res = atb(i);
65-
if (res == _NA) nc._xs[i] = Integer.MIN_VALUE;
66-
else nc._ls[i] = res;
64+
if (res == _NA) nc.addNA();
65+
else nc.addNum(res,0);
6766
}
6867
return nc;
6968
}

‎src/main/java/water/fvec/CX0Chunk.java

+8-5
Original file line numberDiff line numberDiff line change
@@ -17,11 +17,14 @@ public final class CX0Chunk extends CXIChunk {
1717
@Override boolean hasFloat () { return false; }
1818

1919
@Override NewChunk inflate_impl(NewChunk nc) {
20-
final int len = sparseLen();
21-
nc._ls = MemoryManager.malloc8 (len);
22-
Arrays.fill(nc._ls,1);
23-
nc._xs = MemoryManager.malloc4 (len);
24-
nc._id = nonzeros();
20+
final int slen = sparseLen();
21+
nc.set_sparseLen(slen);
22+
nc.set_len(len());
23+
nc.alloc_mantissa(slen);
24+
Arrays.fill(nc.mantissa(),1);
25+
nc.alloc_exponent(slen);
26+
nc.alloc_indices(slen);
27+
nonzeros(nc.indices());
2528
return nc;
2629
}
2730

0 commit comments

Comments
 (0)