Skip to content

Commit

Permalink
Merge pull request orbisgis#1109 from nicolas-f/faster_driver
Browse files Browse the repository at this point in the history 
Faster SHP and DBF file table
  • Loading branch information
@ebocher
ebocher authored Jul 23, 2020
2 parents 91e8382 + 7ece9f1 commit 621aa61
Show file tree
Hide file tree
Showing 9 changed files with 183 additions and 78 deletions.
13 changes: 12 additions & 1 deletion h2gis-api/src/main/java/org/h2gis/api/FileDriver.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,17 @@ public interface FileDriver {
*/
long getRowCount();

/**
* @return Estimated row length in bytes
*/
int getEstimatedRowSize(long rowId);

/**
*
* @return Column count
*/
int getFieldCount();

/**
* Close the file, free resources.
*
Expand All @@ -58,7 +69,7 @@ public interface FileDriver {
* @return The row content.
* @throws java.io.IOException Read error.
*/
Object[] getRow(long rowId) throws IOException;
Object getField(long rowId, int columnId) throws IOException;

/**
* Insert values to the current row.
Expand Down
6 changes: 3 additions & 3 deletions h2gis-functions/src/main/java/org/h2gis/functions/io/dbf/DBFDriverFunction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -245,16 +245,16 @@ public void importFile(Connection connection, String tableReference, File fileNa
}
try {
connection.setAutoCommit(false);
int columnCount = dbfDriver.getFieldCount();
try (PreparedStatement preparedStatement = connection.prepareStatement(
String.format("INSERT INTO %s VALUES ( %s )", parsedTable,
getQuestionMark(dbfHeader.getNumFields() + 1)))) {
JDBCUtilities.attachCancelResultSet(preparedStatement, progress);
long batchSize = 0;
for (int rowId = 0; rowId < dbfDriver.getRowCount(); rowId++) {
preparedStatement.setObject(1, rowId + 1);
Value[] values = dbfDriver.getRow(rowId);
for (int columnId = 0; columnId < values.length; columnId++) {
preparedStatement.setObject(columnId + 2, values[columnId].getObject());
for (int columnId = 0; columnId < columnCount; columnId++) {
preparedStatement.setObject(columnId + 2, dbfDriver.getField(rowId, columnId).getObject());
}
preparedStatement.addBatch();
batchSize++;
Expand Down
17 changes: 11 additions & 6 deletions h2gis-functions/src/main/java/org/h2gis/functions/io/dbf/internal/DBFDriver.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -132,18 +132,23 @@ public long getRowCount() {
/**
* @return Column count
*/
@Override
public int getFieldCount() {
return getDbaseFileHeader().getNumFields();
}

@Override
public Value[] getRow(long rowId) throws IOException {
final int fieldCount = dbaseFileReader.getFieldCount();
Value[] values = new Value[fieldCount];
for(int fieldId=0;fieldId<fieldCount;fieldId++) {
values[fieldId] = dbaseFileReader.getFieldValue((int)rowId, fieldId);
public int getEstimatedRowSize(long rowId) {
int totalSize = 0;
for(int column = 0; column < getFieldCount(); column++) {
totalSize += dbaseFileReader.getLengthFor(column);
}
return values;
return totalSize;
}

@Override
public Value getField(long rowId, int columnId) throws IOException {
return dbaseFileReader.getFieldValue((int)rowId, columnId);
}

/**
Expand Down
2 changes: 1 addition & 1 deletion h2gis-functions/src/main/java/org/h2gis/functions/io/dbf/internal/DbaseFileReader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -165,7 +165,7 @@ public Value getFieldValue(int row, int column) throws IOException {

}

private int getLengthFor(int column) {
public int getLengthFor(int column) {
return header.getFieldLength(column);
}

Expand Down
110 changes: 99 additions & 11 deletions h2gis-functions/src/main/java/org/h2gis/functions/io/file_table/H2TableIndex.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@

import org.h2.api.ErrorCode;
import org.h2.command.dml.AllColumnsForPlan;
import org.h2.engine.Constants;
import org.h2.engine.Session;
import org.h2.index.BaseIndex;
import org.h2.index.Cursor;
Expand Down Expand Up @@ -87,17 +88,7 @@ public FileDriver getDriver() {

@Override
public Row getRow(Session session, long key) {
try {
Object[] driverRow = driver.getRow(key - 1);
Value[] values = new Value[driverRow.length + 1];
System.arraycopy(driverRow, 0, values, 1, driverRow.length);
values[0] = ValueBigint.get(key);
Row row = Row.get(values, Row.MEMORY_CALCULATE);
row.setKey(key);
return row;
} catch (IOException ex) {
throw DbException.get(ErrorCode.IO_EXCEPTION_1,ex);
}
return new DriverRow(driver, key);
}

@Override
Expand Down Expand Up @@ -254,4 +245,101 @@ public boolean previous() {
}
}
}

/**
* This class is requiring only field value on demand instead of gathering the full row values from drivers
*/
public static class DriverRow extends Row {
FileDriver driver;
int memory; // estimated row size in bytes

public DriverRow(FileDriver driver, long key) {
this.driver = driver;
this.key = key;
}

@Override
public Value[] getValueList() {
try {
Value[] values = new Value[getColumnCount()];
values[0] = ValueBigint.get(key);
for(int i = 1; i < values.length; i++) {
values[i] = (Value)(driver.getField(key - 1, i - 1));
}
return values;
} catch (IOException ex) {
throw DbException.get(ErrorCode.IO_EXCEPTION_1, ex);
}
}

@Override
public int getColumnCount() {
return driver.getFieldCount() + 1;
}

@Override
public Value getValue(int column) {
if(column == ROWID_INDEX) {
return ValueBigint.get(key);
} else {
try {
if(column == 0) {
// pk
return ValueBigint.get(key);
} else {
return (Value)(driver.getField(key - 1, column - 1));
}
} catch (IOException ex) {
throw DbException.get(ErrorCode.IO_EXCEPTION_1,ex);
}
}
}

@Override
public void setValue(int i, Value value) {
if (i == ROWID_INDEX) {
key = value.getLong();
}
}

@Override
public int getMemory() {
if (memory != MEMORY_CALCULATE) {
return memory;
}
return memory = calculateMemory();
}

@Override
public void copyFrom(SearchRow source) {
setKey(source.getKey());
for (int i = 0; i < getColumnCount(); i++) {
setValue(i, source.getValue(i));
}
}

@Override
public String toString() {
StringBuilder builder = new StringBuilder("( /* key:").append(key).append(" */ ");
for (int i = 0, length = getColumnCount(); i < length; i++) {
if (i > 0) {
builder.append(", ");
}
Value v = getValue(i);
builder.append(v == null ? "null" : v.getTraceSQL());
}
return builder.append(')').toString();
}

/**
* Calculate the estimated memory used for this row, in bytes.
*
* @return the memory
*/
int calculateMemory() {
int m = Constants.MEMORY_ROW + Constants.MEMORY_ARRAY + getColumnCount() * Constants.MEMORY_POINTER;
m += driver.getEstimatedRowSize(key - 1);
return m;
}
}
}
6 changes: 3 additions & 3 deletions h2gis-functions/src/main/java/org/h2gis/functions/io/shp/SHPDriverFunction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -308,12 +308,12 @@ public void importFile(Connection connection, String tableReference, File fileNa
lastSql = String.format("INSERT INTO %s VALUES (DEFAULT, %s )", parse,
DBFDriverFunction.getQuestionMark(dbfHeader.getNumFields() + 1));
connection.setAutoCommit(false);
final int columnCount = shpDriver.getFieldCount();
try (PreparedStatement preparedStatement = connection.prepareStatement(lastSql)) {
long batchSize = 0;
for (int rowId = 0; rowId < shpDriver.getRowCount(); rowId++) {
Value[] values = shpDriver.getRow(rowId);
for (int columnId = 0; columnId < values.length; columnId++) {
preparedStatement.setObject(columnId + 1, values[columnId].getObject());
for (int columnId = 0; columnId < columnCount; columnId++) {
preparedStatement.setObject(columnId + 1, shpDriver.getField(rowId, columnId).getObject());
}
preparedStatement.addBatch();
batchSize++;
Expand Down
43 changes: 28 additions & 15 deletions h2gis-functions/src/main/java/org/h2gis/functions/io/shp/internal/SHPDriver.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@

import org.h2.value.Value;
import org.h2.value.ValueGeometry;
import org.h2.value.ValueNull;
import org.h2gis.api.FileDriver;
import org.h2gis.functions.io.dbf.internal.DBFDriver;
import org.h2gis.functions.io.dbf.internal.DbaseFileHeader;
Expand All @@ -39,7 +40,7 @@
* How to use:
*
* In Write mode,
* Declare fields by calling {@link SHPDriver#initDriver(java.io.File, ShapeType, org.h2gis.drivers.dbf.internal.DbaseFileHeader)}
* Declare fields by calling {@link SHPDriver#initDriver(File, ShapeType, DbaseFileHeader)} Driver(java.io.File, ShapeType, org.h2gis.drivers.dbf.internal.DbaseFileHeader)}
* then write row using
*
*
Expand Down Expand Up @@ -222,28 +223,40 @@ public void close() throws IOException {
}
}

@Override
public int getFieldCount() {
return dbfDriver.getFieldCount() + 1;
}

@Override
public Value[] getRow(long rowId) throws IOException {
final int fieldCount = getFieldCount();
Value[] values = new Value[fieldCount];
int deltaDBF = 0;
for (int i = 0; i < fieldCount; i++) {
if (i == geometryFieldIndex) {
Geometry geom = shapefileReader.geomAt(shxFileReader.getOffset((int) rowId));
if (geom != null) {
geom.setSRID(getSrid());
}
values[i] = ValueGeometry.getFromGeometry(geom);
public int getEstimatedRowSize(long rowId) {
int totalSize = 0;
totalSize += dbfDriver.getEstimatedRowSize(rowId);
try {
totalSize += shxFileReader.getContentLength((int) rowId);
} catch (IOException ex) {
// Ignore
}
return totalSize;
}

@Override
public Value getField(long rowId, int column) throws IOException {
if (column == geometryFieldIndex) {
Geometry geom = shapefileReader.geomAt(shxFileReader.getOffset((int) rowId));
if (geom != null) {
geom.setSRID(getSrid());
return ValueGeometry.getFromGeometry(geom);
} else {
return ValueNull.INSTANCE;
}
} else {
if(geometryFieldIndex < column) {
return dbfDriver.getDbaseFileReader().getFieldValue((int) rowId, column - 1);
} else {
values[i] = dbfDriver.getDbaseFileReader().getFieldValue((int) rowId, deltaDBF);
deltaDBF++;
return dbfDriver.getDbaseFileReader().getFieldValue((int) rowId, column);
}
}
return values;
}

/**
Expand Down
14 changes: 6 additions & 8 deletions h2gis-functions/src/test/java/org/h2gis/functions/io/dbf/DBFImportExportTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,14 +84,12 @@ public void exportTableTestGeomEnd() throws SQLException, IOException {
dbfDriver.initDriverFromFile(dbfFile);
assertEquals(3, dbfDriver.getFieldCount());
assertEquals(2, dbfDriver.getRowCount());
Value[] row = dbfDriver.getRow(0);
assertEquals(1, row[0].getInt());
assertEquals(4.9406564584124654, row[1].getDouble(), 1e-12);
assertEquals("main area", row[2].getString());
row = dbfDriver.getRow(1);
assertEquals(2, row[0].getInt());
assertEquals(2.2250738585072009, row[1].getDouble(), 1e-12);
assertEquals("second area", row[2].getString());
assertEquals(1, dbfDriver.getField(0, 0).getInt());
assertEquals(4.9406564584124654, dbfDriver.getField(0, 1).getDouble(), 1e-12);
assertEquals("main area", dbfDriver.getField(0, 2).getString());
assertEquals(2, dbfDriver.getField(1, 0).getInt());
assertEquals(2.2250738585072009, dbfDriver.getField(1, 1).getDouble(), 1e-12);
assertEquals("second area", dbfDriver.getField(1,2).getString());
}

@Test
Expand Down
Loading

0 comments on commit 621aa61

Please sign in to comment.