Skip to content

Commit

Permalink
Merge remote-tracking branch 'upstream/mirror' into mirror
Browse files Browse the repository at this point in the history
  • Loading branch information
@wiktorn
wiktorn committed Nov 18, 2018
2 parents 3397f30 + 2ae1a36 commit dada6eb
Show file tree
Hide file tree
Showing 7 changed files with 107 additions and 60 deletions.
46 changes: 31 additions & 15 deletions src/org/apache/commons/compress/archivers/zip/ParallelScatterZipCreator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -239,28 +239,32 @@ public Object call() throws Exception {
public void writeTo(final ZipArchiveOutputStream targetStream)
throws IOException, InterruptedException, ExecutionException {

// Make sure we catch any exceptions from parallel phase
try {
for (final Future<?> future : futures) {
future.get();
// Make sure we catch any exceptions from parallel phase
try {
for (final Future<?> future : futures) {
future.get();
}
} finally {
es.shutdown();
}
} finally {
es.shutdown();
}

es.awaitTermination(1000 * 60L, TimeUnit.SECONDS); // == Infinity. We really *must* wait for this to complete
es.awaitTermination(1000 * 60L, TimeUnit.SECONDS); // == Infinity. We really *must* wait for this to complete

// It is important that all threads terminate before we go on, ensure happens-before relationship
compressionDoneAt = System.currentTimeMillis();
// It is important that all threads terminate before we go on, ensure happens-before relationship
compressionDoneAt = System.currentTimeMillis();

synchronized (streams) {
for (final ScatterZipOutputStream scatterStream : streams) {
scatterStream.writeTo(targetStream);
scatterStream.close();
synchronized (streams) {
for (final ScatterZipOutputStream scatterStream : streams) {
scatterStream.writeTo(targetStream);
scatterStream.close();
}
}
}

scatterDoneAt = System.currentTimeMillis();
scatterDoneAt = System.currentTimeMillis();
} finally {
ensureStreamsAreClosed();
}
}

/**
Expand All @@ -271,5 +275,17 @@ public void writeTo(final ZipArchiveOutputStream targetStream)
public ScatterStatistics getStatisticsMessage() {
return new ScatterStatistics(compressionDoneAt - startedAt, scatterDoneAt - compressionDoneAt);
}

private void ensureStreamsAreClosed() {
synchronized (streams) {
for (final ScatterZipOutputStream scatterStream : streams) {
try {
scatterStream.close();
} catch (IOException ex) {
// no way to properly log this
}
}
}
}
}

5 changes: 5 additions & 0 deletions src/org/apache/commons/compress/archivers/zip/ScatterZipOutputStream.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
import java.io.InputStream;
import java.util.Queue;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.zip.Deflater;

/**
Expand All @@ -49,6 +50,7 @@ public class ScatterZipOutputStream implements Closeable {
private final Queue<CompressedEntry> items = new ConcurrentLinkedQueue<>();
private final ScatterGatherBackingStore backingStore;
private final StreamCompressor streamCompressor;
private AtomicBoolean isClosed = new AtomicBoolean();

private static class CompressedEntry {
final ZipArchiveEntryRequest zipArchiveEntryRequest;
Expand Down Expand Up @@ -124,6 +126,9 @@ public void writeTo(final ZipArchiveOutputStream target) throws IOException {
*/
@Override
public void close() throws IOException {
if (!isClosed.compareAndSet(false, true)) {
return;
}
try {
backingStore.close();
} finally {
Expand Down
4 changes: 3 additions & 1 deletion src/org/apache/commons/compress/parallel/FileBasedScatterGatherBackingStore.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,9 @@ public void close() throws IOException {
try {
closeForWriting();
} finally {
target.delete();
if (target.exists() && !target.delete()) {
target.deleteOnExit();
}
}
}
}
2 changes: 2 additions & 0 deletions src/org/openstreetmap/josm/actions/JoinAreasAction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1378,6 +1378,8 @@ public static List<Multipolygon> collectMultipolygons(Collection<Way> selectedWa
innerWays.clear();

for (RelationMember rm : r.getMembers()) {
if (!rm.isWay())
continue;
if ("outer".equalsIgnoreCase(rm.getRole())) {
outerWays.add(rm.getWay());
hasKnownOuter |= selectedWays.contains(rm.getWay());
Expand Down
84 changes: 50 additions & 34 deletions src/org/openstreetmap/josm/gui/layer/AbstractTileSourceLayer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1769,20 +1769,47 @@ public void visitBoundingBox(BoundingXYVisitor v) {
*/
public class PrecacheTask implements TileLoaderListener {
private final ProgressMonitor progressMonitor;
private int totalCount;
private final int totalCount;
private final AtomicInteger processedCount = new AtomicInteger(0);
private final TileLoader tileLoader;
private final Set<Tile> requestedTiles;

/**
* @param progressMonitor that will be notified about progess of the task
* @param bufferY buffer Y in degrees around which to download tiles
* @param bufferX buffer X in degrees around which to download tiles
* @param points list of points along which to download
*/
public PrecacheTask(ProgressMonitor progressMonitor) {
public PrecacheTask(ProgressMonitor progressMonitor, List<LatLon> points, double bufferX, double bufferY) {
this.progressMonitor = progressMonitor;
this.tileLoader = getTileLoaderFactory().makeTileLoader(this, getHeaders(tileSource), minimumTileExpire);
if (this.tileLoader instanceof TMSCachedTileLoader) {
((TMSCachedTileLoader) this.tileLoader).setDownloadExecutor(
TMSCachedTileLoader.getNewThreadPoolExecutor("Precache downloader"));
}
requestedTiles = new ConcurrentSkipListSet<>(
(o1, o2) -> String.CASE_INSENSITIVE_ORDER.compare(o1.getKey(), o2.getKey()));
for (LatLon point: points) {
TileXY minTile = tileSource.latLonToTileXY(point.lat() - bufferY, point.lon() - bufferX, currentZoomLevel);
TileXY curTile = tileSource.latLonToTileXY(CoordinateConversion.llToCoor(point), currentZoomLevel);
TileXY maxTile = tileSource.latLonToTileXY(point.lat() + bufferY, point.lon() + bufferX, currentZoomLevel);

// take at least one tile of buffer
int minY = Math.min(curTile.getYIndex() - 1, minTile.getYIndex());
int maxY = Math.max(curTile.getYIndex() + 1, maxTile.getYIndex());
int minX = Math.min(curTile.getXIndex() - 1, minTile.getXIndex());
int maxX = Math.max(curTile.getXIndex() + 1, maxTile.getXIndex());

for (int x = minX; x <= maxX; x++) {
for (int y = minY; y <= maxY; y++) {
requestedTiles.add(new Tile(tileSource, x, y, currentZoomLevel));
}
}
}

this.totalCount = requestedTiles.size();
this.progressMonitor.setTicksCount(requestedTiles.size());

}

/**
Expand Down Expand Up @@ -1812,8 +1839,12 @@ public void cancel() {
public void tileLoadingFinished(Tile tile, boolean success) {
int processed = this.processedCount.incrementAndGet();
if (success) {
this.progressMonitor.worked(1);
this.progressMonitor.setCustomText(tr("Downloaded {0}/{1} tiles", processed, totalCount));
synchronized (progressMonitor) {
if (!this.progressMonitor.isCanceled()) {
this.progressMonitor.worked(1);
this.progressMonitor.setCustomText(tr("Downloaded {0}/{1} tiles", processed, totalCount));
}
}
} else {
Logging.warn("Tile loading failure: " + tile + " - " + tile.getErrorMessage());
}
Expand All @@ -1825,6 +1856,19 @@ public void tileLoadingFinished(Tile tile, boolean success) {
public TileLoader getTileLoader() {
return tileLoader;
}

/**
* Execute the download
*/
public void run() {
TileLoader loader = getTileLoader();
for (Tile t: requestedTiles) {
if (!progressMonitor.isCanceled()) {
loader.createTileLoaderJob(t).submit();
}
}

}
}

/**
Expand All @@ -1839,37 +1883,9 @@ public TileLoader getTileLoader() {
* @param bufferY how many units in current Coordinate Reference System to cover in Y axis in both sides
* @return precache task representing download task
*/
public AbstractTileSourceLayer<T>.PrecacheTask downloadAreaToCache(final ProgressMonitor progressMonitor, List<LatLon> points,
public AbstractTileSourceLayer<T>.PrecacheTask getDownloadAreaToCacheTask(final ProgressMonitor progressMonitor, List<LatLon> points,
double bufferX, double bufferY) {
PrecacheTask precacheTask = new PrecacheTask(progressMonitor);
final Set<Tile> requestedTiles = new ConcurrentSkipListSet<>(
(o1, o2) -> String.CASE_INSENSITIVE_ORDER.compare(o1.getKey(), o2.getKey()));
for (LatLon point: points) {
TileXY minTile = tileSource.latLonToTileXY(point.lat() - bufferY, point.lon() - bufferX, currentZoomLevel);
TileXY curTile = tileSource.latLonToTileXY(CoordinateConversion.llToCoor(point), currentZoomLevel);
TileXY maxTile = tileSource.latLonToTileXY(point.lat() + bufferY, point.lon() + bufferX, currentZoomLevel);

// take at least one tile of buffer
int minY = Math.min(curTile.getYIndex() - 1, minTile.getYIndex());
int maxY = Math.max(curTile.getYIndex() + 1, maxTile.getYIndex());
int minX = Math.min(curTile.getXIndex() - 1, minTile.getXIndex());
int maxX = Math.max(curTile.getXIndex() + 1, maxTile.getXIndex());

for (int x = minX; x <= maxX; x++) {
for (int y = minY; y <= maxY; y++) {
requestedTiles.add(new Tile(tileSource, x, y, currentZoomLevel));
}
}
}

precacheTask.totalCount = requestedTiles.size();
precacheTask.progressMonitor.setTicksCount(requestedTiles.size());

TileLoader loader = precacheTask.getTileLoader();
for (Tile t: requestedTiles) {
loader.createTileLoaderJob(t).submit();
}
return precacheTask;
return new PrecacheTask(progressMonitor, points, bufferX, bufferY);
}

@Override
Expand Down
23 changes: 14 additions & 9 deletions src/org/openstreetmap/josm/gui/layer/OsmDataLayer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,7 @@
import org.openstreetmap.josm.tools.ImageProvider;
import org.openstreetmap.josm.tools.ImageProvider.ImageSizes;
import org.openstreetmap.josm.tools.Logging;
import org.openstreetmap.josm.tools.UncheckedParseException;
import org.openstreetmap.josm.tools.date.DateUtils;

/**
Expand Down Expand Up @@ -795,15 +796,19 @@ public static WayPoint nodeToWayPoint(Node n, long time) {

addDoubleIfPresent(wpt, n, GpxConstants.PT_ELE);

if (time > 0) {
wpt.put(GpxConstants.PT_TIME, DateUtils.fromTimestamp(time));
wpt.setTime(time);
} else if (n.hasKey(GpxConstants.PT_TIME)) {
wpt.put(GpxConstants.PT_TIME, DateUtils.fromString(n.get(GpxConstants.PT_TIME)));
wpt.setTime();
} else if (!n.isTimestampEmpty()) {
wpt.put(GpxConstants.PT_TIME, DateUtils.fromTimestamp(n.getRawTimestamp()));
wpt.setTime();
try {
if (time > 0) {
wpt.put(GpxConstants.PT_TIME, DateUtils.fromTimestamp(time));
wpt.setTime(time);
} else if (n.hasKey(GpxConstants.PT_TIME)) {
wpt.put(GpxConstants.PT_TIME, DateUtils.fromString(n.get(GpxConstants.PT_TIME)));
wpt.setTime();
} else if (!n.isTimestampEmpty()) {
wpt.put(GpxConstants.PT_TIME, DateUtils.fromTimestamp(n.getRawTimestamp()));
wpt.setTime();
}
} catch (UncheckedParseException e) {
Logging.error(e);
}

addDoubleIfPresent(wpt, n, GpxConstants.PT_MAGVAR);
Expand Down
3 changes: 2 additions & 1 deletion src/org/openstreetmap/josm/gui/layer/gpx/DownloadWmsAlongTrackAction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,8 @@ protected PrecacheWmsTask(AbstractTileSourceLayer<? extends AbstractTMSTileSourc

@Override
protected void realRun() throws SAXException, IOException, OsmTransferException {
precacheTask = layer.downloadAreaToCache(progressMonitor, points, 0, 0);
precacheTask = layer.getDownloadAreaToCacheTask(progressMonitor, points, 0, 0);
precacheTask.run();
synchronized (this) {
try {
while (!precacheTask.isFinished() && !progressMonitor.isCanceled()) {
Expand Down

0 comments on commit dada6eb

Please sign in to comment.