Skip to content

Commit

Permalink
Merge pull request Alluxio#4877 from maobaolong/mbl-freeRatio
Browse files Browse the repository at this point in the history
[ALLUXIO-2653,ALLUXIO-1201] Use two watermarks for space reserver
yupeng9 authored Apr 25, 2017
2 parents 5173e18 + f2a39eb commit 457857f
Showing 15 changed files with 276 additions and 42 deletions.
65 changes: 64 additions & 1 deletion core/common/src/main/java/alluxio/PropertyKey.java
Original file line number Diff line number Diff line change
@@ -229,19 +229,50 @@ public enum PropertyKey {
WORKER_TIERED_STORE_LEVEL0_DIRS_PATH(Name.WORKER_TIERED_STORE_LEVEL0_DIRS_PATH, "/mnt/ramdisk"),
WORKER_TIERED_STORE_LEVEL0_DIRS_QUOTA(Name.WORKER_TIERED_STORE_LEVEL0_DIRS_QUOTA,
"${alluxio.worker.memory.size}"),
WORKER_TIERED_STORE_LEVEL0_RESERVED_RATIO(Name.WORKER_TIERED_STORE_LEVEL0_RESERVED_RATIO, "0.1"),
/**
* @deprecated It will be removed in 2.0.0.
* Use {@link #WORKER_TIERED_STORE_LEVEL0_HIGH_WATERMARK_RATIO} and
* {@link #WORKER_TIERED_STORE_LEVEL0_LOW_WATERMARK_RATIO} instead.
*/
@Deprecated
WORKER_TIERED_STORE_LEVEL0_RESERVED_RATIO(Name.WORKER_TIERED_STORE_LEVEL0_RESERVED_RATIO, null),
WORKER_TIERED_STORE_LEVEL0_HIGH_WATERMARK_RATIO(
Name.WORKER_TIERED_STORE_LEVEL0_HIGH_WATERMARK_RATIO, 1.0),
WORKER_TIERED_STORE_LEVEL0_LOW_WATERMARK_RATIO(
Name.WORKER_TIERED_STORE_LEVEL0_LOW_WATERMARK_RATIO, 0.7),
WORKER_TIERED_STORE_LEVEL1_ALIAS(Name.WORKER_TIERED_STORE_LEVEL1_ALIAS, null),
WORKER_TIERED_STORE_LEVEL1_DIRS_PATH(Name.WORKER_TIERED_STORE_LEVEL1_DIRS_PATH, null),
WORKER_TIERED_STORE_LEVEL1_DIRS_QUOTA(Name.WORKER_TIERED_STORE_LEVEL1_DIRS_QUOTA, null),
/**
* @deprecated It will be removed in 2.0.0.
* Use {@link #WORKER_TIERED_STORE_LEVEL1_HIGH_WATERMARK_RATIO} and
* {@link #WORKER_TIERED_STORE_LEVEL1_LOW_WATERMARK_RATIO} instead.
*/
@Deprecated
WORKER_TIERED_STORE_LEVEL1_RESERVED_RATIO(Name.WORKER_TIERED_STORE_LEVEL1_RESERVED_RATIO, null),
WORKER_TIERED_STORE_LEVEL1_HIGH_WATERMARK_RATIO(
Name.WORKER_TIERED_STORE_LEVEL1_HIGH_WATERMARK_RATIO, 1.0),
WORKER_TIERED_STORE_LEVEL1_LOW_WATERMARK_RATIO(
Name.WORKER_TIERED_STORE_LEVEL1_LOW_WATERMARK_RATIO, 0.7),
WORKER_TIERED_STORE_LEVEL2_ALIAS(Name.WORKER_TIERED_STORE_LEVEL2_ALIAS, null),
WORKER_TIERED_STORE_LEVEL2_DIRS_PATH(Name.WORKER_TIERED_STORE_LEVEL2_DIRS_PATH, null),
WORKER_TIERED_STORE_LEVEL2_DIRS_QUOTA(Name.WORKER_TIERED_STORE_LEVEL2_DIRS_QUOTA, null),
/**
* @deprecated It will be removed in 2.0.0.
* Use {@link #WORKER_TIERED_STORE_LEVEL2_HIGH_WATERMARK_RATIO} and
* {@link #WORKER_TIERED_STORE_LEVEL2_LOW_WATERMARK_RATIO} instead.
*/
@Deprecated
WORKER_TIERED_STORE_LEVEL2_RESERVED_RATIO(Name.WORKER_TIERED_STORE_LEVEL2_RESERVED_RATIO, null),
WORKER_TIERED_STORE_LEVEL2_HIGH_WATERMARK_RATIO(
Name.WORKER_TIERED_STORE_LEVEL2_HIGH_WATERMARK_RATIO, 1.0),
WORKER_TIERED_STORE_LEVEL2_LOW_WATERMARK_RATIO(
Name.WORKER_TIERED_STORE_LEVEL2_LOW_WATERMARK_RATIO, 0.7),
WORKER_TIERED_STORE_LEVELS(Name.WORKER_TIERED_STORE_LEVELS, 1),
WORKER_TIERED_STORE_RESERVER_ENABLED(Name.WORKER_TIERED_STORE_RESERVER_ENABLED, false),
WORKER_TIERED_STORE_RESERVER_INTERVAL_MS(Name.WORKER_TIERED_STORE_RESERVER_INTERVAL_MS, 1000),
WORKER_TIERED_STORE_RETRY(Name.WORKER_TIERED_STORE_RETRY, 3),
WORKER_TIERED_STORE_FREE_SPACE_RATIO(Name.WORKER_TIERED_STORE_FREE_SPACE_RATIO, 0.0f),
WORKER_WEB_BIND_HOST(Name.WORKER_WEB_BIND_HOST, "0.0.0.0"),
WORKER_WEB_HOSTNAME(Name.WORKER_WEB_HOSTNAME, null),
WORKER_WEB_PORT(Name.WORKER_WEB_PORT, 30000),
@@ -675,30 +706,62 @@ public static final class Name {
"alluxio.worker.tieredstore.level0.dirs.path";
public static final String WORKER_TIERED_STORE_LEVEL0_DIRS_QUOTA =
"alluxio.worker.tieredstore.level0.dirs.quota";
/**
* @deprecated It will be removed in 2.0.0.
* Use {@link #WORKER_TIERED_STORE_LEVEL0_HIGH_WATERMARK_RATIO} and
* {@link #WORKER_TIERED_STORE_LEVEL0_LOW_WATERMARK_RATIO} instead.
*/
@Deprecated
public static final String WORKER_TIERED_STORE_LEVEL0_RESERVED_RATIO =
"alluxio.worker.tieredstore.level0.reserved.ratio";
public static final String WORKER_TIERED_STORE_LEVEL0_HIGH_WATERMARK_RATIO =
"alluxio.worker.tieredstore.level0.watermark.high.ratio";
public static final String WORKER_TIERED_STORE_LEVEL0_LOW_WATERMARK_RATIO =
"alluxio.worker.tieredstore.level0.watermark.low.ratio";
public static final String WORKER_TIERED_STORE_LEVEL1_ALIAS =
"alluxio.worker.tieredstore.level1.alias";
public static final String WORKER_TIERED_STORE_LEVEL1_DIRS_PATH =
"alluxio.worker.tieredstore.level1.dirs.path";
public static final String WORKER_TIERED_STORE_LEVEL1_DIRS_QUOTA =
"alluxio.worker.tieredstore.level1.dirs.quota";
/**
* @deprecated It will be removed in 2.0.0.
* Use {@link #WORKER_TIERED_STORE_LEVEL1_HIGH_WATERMARK_RATIO} and
* {@link #WORKER_TIERED_STORE_LEVEL1_LOW_WATERMARK_RATIO} instead.
*/
@Deprecated
public static final String WORKER_TIERED_STORE_LEVEL1_RESERVED_RATIO =
"alluxio.worker.tieredstore.level1.reserved.ratio";
public static final String WORKER_TIERED_STORE_LEVEL1_HIGH_WATERMARK_RATIO =
"alluxio.worker.tieredstore.level1.watermark.high.ratio";
public static final String WORKER_TIERED_STORE_LEVEL1_LOW_WATERMARK_RATIO =
"alluxio.worker.tieredstore.level1.watermark.low.ratio";
public static final String WORKER_TIERED_STORE_LEVEL2_ALIAS =
"alluxio.worker.tieredstore.level2.alias";
public static final String WORKER_TIERED_STORE_LEVEL2_DIRS_PATH =
"alluxio.worker.tieredstore.level2.dirs.path";
public static final String WORKER_TIERED_STORE_LEVEL2_DIRS_QUOTA =
"alluxio.worker.tieredstore.level2.dirs.quota";
/**
* @deprecated It will be removed in 2.0.0.
* Use {@link #WORKER_TIERED_STORE_LEVEL2_HIGH_WATERMARK_RATIO} and
* {@link #WORKER_TIERED_STORE_LEVEL2_LOW_WATERMARK_RATIO} instead.
*/
@Deprecated
public static final String WORKER_TIERED_STORE_LEVEL2_RESERVED_RATIO =
"alluxio.worker.tieredstore.level2.reserved.ratio";
public static final String WORKER_TIERED_STORE_LEVEL2_HIGH_WATERMARK_RATIO =
"alluxio.worker.tieredstore.level2.watermark.high.ratio";
public static final String WORKER_TIERED_STORE_LEVEL2_LOW_WATERMARK_RATIO =
"alluxio.worker.tieredstore.level2.watermark.low.ratio";
public static final String WORKER_TIERED_STORE_LEVELS = "alluxio.worker.tieredstore.levels";
public static final String WORKER_TIERED_STORE_RESERVER_ENABLED =
"alluxio.worker.tieredstore.reserver.enabled";
public static final String WORKER_TIERED_STORE_RESERVER_INTERVAL_MS =
"alluxio.worker.tieredstore.reserver.interval.ms";
public static final String WORKER_TIERED_STORE_RETRY = "alluxio.worker.tieredstore.retry";
public static final String WORKER_TIERED_STORE_FREE_SPACE_RATIO
= "alluxio.worker.tieredstore.free.space.ratio";
public static final String WORKER_WEB_BIND_HOST = "alluxio.worker.web.bind.host";
public static final String WORKER_WEB_HOSTNAME = "alluxio.worker.web.hostname";
public static final String WORKER_WEB_PORT = "alluxio.worker.web.port";
12 changes: 11 additions & 1 deletion core/common/src/main/java/alluxio/PropertyKeyFormat.java
Original file line number Diff line number Diff line change
@@ -24,6 +24,16 @@ public enum PropertyKeyFormat {
WORKER_TIERED_STORE_LEVEL_ALIAS_FORMAT("alluxio.worker.tieredstore.level%d.alias"),
WORKER_TIERED_STORE_LEVEL_DIRS_PATH_FORMAT("alluxio.worker.tieredstore.level%d.dirs.path"),
WORKER_TIERED_STORE_LEVEL_DIRS_QUOTA_FORMAT("alluxio.worker.tieredstore.level%d.dirs.quota"),
WORKER_TIERED_STORE_LEVEL_HIGH_WATERMARK_RATIO_FORMAT(
"alluxio.worker.tieredstore.level%d.watermark.high.ratio"),
WORKER_TIERED_STORE_LEVEL_LOW_WATERMARK_RATIO_FORMAT(
"alluxio.worker.tieredstore.level%d.watermark.low.ratio"),
/**
* @deprecated It will be removed in 2.0.0.
* Use {@link #WORKER_TIERED_STORE_LEVEL_HIGH_WATERMARK_RATIO_FORMAT} and
* {@link #WORKER_TIERED_STORE_LEVEL_LOW_WATERMARK_RATIO_FORMAT} instead.
*/
@Deprecated
WORKER_TIERED_STORE_LEVEL_RESERVED_RATIO_FORMAT(
"alluxio.worker.tieredstore.level%d.reserved.ratio"),
;
@@ -46,7 +56,7 @@ public String toString() {

/**
* Converts a property key template (e.g.,
* {@link PropertyKeyFormat#WORKER_TIERED_STORE_LEVEL_RESERVED_RATIO_FORMAT})
* {@link PropertyKeyFormat#WORKER_TIERED_STORE_LEVEL_DIRS_PATH_FORMAT})
* to a {@link PropertyKey} enum instance.
*
* @param params ordinal
12 changes: 6 additions & 6 deletions core/common/src/test/java/alluxio/PropertyKeyFormatTest.java
Original file line number Diff line number Diff line change
@@ -61,11 +61,11 @@ public void formatWorkerTieredStoreDirsQuota() throws Exception {

@Test
public void formatWorkerTieredStoreReservedRatio() throws Exception {
Assert.assertEquals(PropertyKey.WORKER_TIERED_STORE_LEVEL0_RESERVED_RATIO,
PropertyKeyFormat.WORKER_TIERED_STORE_LEVEL_RESERVED_RATIO_FORMAT.format(0));
Assert.assertEquals(PropertyKey.WORKER_TIERED_STORE_LEVEL1_RESERVED_RATIO,
PropertyKeyFormat.WORKER_TIERED_STORE_LEVEL_RESERVED_RATIO_FORMAT.format(1));
Assert.assertEquals(PropertyKey.WORKER_TIERED_STORE_LEVEL2_RESERVED_RATIO,
PropertyKeyFormat.WORKER_TIERED_STORE_LEVEL_RESERVED_RATIO_FORMAT.format(2));
Assert.assertEquals(PropertyKey.WORKER_TIERED_STORE_LEVEL0_HIGH_WATERMARK_RATIO,
PropertyKeyFormat.WORKER_TIERED_STORE_LEVEL_HIGH_WATERMARK_RATIO_FORMAT.format(0));
Assert.assertEquals(PropertyKey.WORKER_TIERED_STORE_LEVEL1_HIGH_WATERMARK_RATIO,
PropertyKeyFormat.WORKER_TIERED_STORE_LEVEL_HIGH_WATERMARK_RATIO_FORMAT.format(1));
Assert.assertEquals(PropertyKey.WORKER_TIERED_STORE_LEVEL2_HIGH_WATERMARK_RATIO,
PropertyKeyFormat.WORKER_TIERED_STORE_LEVEL_HIGH_WATERMARK_RATIO_FORMAT.format(2));
}
}
Original file line number Diff line number Diff line change
@@ -29,7 +29,6 @@
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;

import javax.annotation.concurrent.NotThreadSafe;

/**
@@ -39,14 +38,16 @@
@NotThreadSafe
public class SpaceReserver implements HeartbeatExecutor {
private static final Logger LOG = LoggerFactory.getLogger(SpaceReserver.class);

private final BlockWorker mBlockWorker;

/** Association between storage tier aliases and ordinals for the worker. */
private final StorageTierAssoc mStorageTierAssoc;

/** Mapping from tier alias to high watermark in bytes. */
private final Map<String, Long> mHighWaterMarkInBytesOnTiers = new HashMap<>();

/** Mapping from tier alias to space size to be reserved on the tier. */
private final Map<String, Long> mBytesToReserveOnTiers = new HashMap<>();
private final Map<String, Long> mReservedBytesOnTiers = new HashMap<>();

/**
* Creates a new instance of {@link SpaceReserver}.
@@ -59,25 +60,61 @@ public SpaceReserver(BlockWorker blockWorker) {
Map<String, Long> capOnTiers = blockWorker.getStoreMeta().getCapacityBytesOnTiers();
long lastTierReservedBytes = 0;
for (int ordinal = 0; ordinal < mStorageTierAssoc.size(); ordinal++) {
PropertyKey tierReservedSpaceProp =
PropertyKeyFormat.WORKER_TIERED_STORE_LEVEL_RESERVED_RATIO_FORMAT.format(ordinal);
String tierAlias = mStorageTierAssoc.getAlias(ordinal);
long reservedSpaceBytes =
(long) (capOnTiers.get(tierAlias) * Configuration.getDouble(tierReservedSpaceProp));
mBytesToReserveOnTiers.put(tierAlias, reservedSpaceBytes + lastTierReservedBytes);
lastTierReservedBytes += reservedSpaceBytes;
long capOnTier = capOnTiers.get(tierAlias);
long reservedBytes;
PropertyKey tierReservedSpaceProp =
PropertyKeyFormat.WORKER_TIERED_STORE_LEVEL_RESERVED_RATIO_FORMAT.format(ordinal);
if (Configuration.containsKey(tierReservedSpaceProp)) {
LOG.warn("The property reserved.ratio is deprecated and high/low water mark "
+ "should be used instead.");
reservedBytes =
(long) (capOnTier * Configuration.getDouble(tierReservedSpaceProp));
} else {
// HighWatemark defines when to start the space reserving process
PropertyKey tierHighWatermarkProp =
PropertyKeyFormat.WORKER_TIERED_STORE_LEVEL_HIGH_WATERMARK_RATIO_FORMAT
.format(ordinal);
long highWatermarkInBytes =
(long) (capOnTier * Configuration.getDouble(tierHighWatermarkProp));

// LowWatemark defines when to stop the space reserving process if started
PropertyKey tierLowWatermarkProp =
PropertyKeyFormat.WORKER_TIERED_STORE_LEVEL_LOW_WATERMARK_RATIO_FORMAT
.format(ordinal);
reservedBytes =
(long) (capOnTier - capOnTier * Configuration.getDouble(tierLowWatermarkProp));
mHighWaterMarkInBytesOnTiers.put(tierAlias, highWatermarkInBytes);
}
mReservedBytesOnTiers.put(tierAlias, reservedBytes + lastTierReservedBytes);
lastTierReservedBytes += reservedBytes;
}
}

private void reserveSpace() {
Map<String, Long> usedBytesOnTiers = mBlockWorker.getStoreMeta().getUsedBytesOnTiers();
for (int ordinal = mStorageTierAssoc.size() - 1; ordinal >= 0; ordinal--) {
String tierAlias = mStorageTierAssoc.getAlias(ordinal);
long bytesReserved = mBytesToReserveOnTiers.get(tierAlias);
try {
mBlockWorker.freeSpace(Sessions.MIGRATE_DATA_SESSION_ID, bytesReserved, tierAlias);
} catch (WorkerOutOfSpaceException | BlockDoesNotExistException | BlockAlreadyExistsException
| InvalidWorkerStateException | IOException e) {
LOG.warn(e.getMessage());
long reservedBytes = mReservedBytesOnTiers.get(tierAlias);
if (mHighWaterMarkInBytesOnTiers.containsKey(tierAlias)) {
long highWatermarkInBytes = mHighWaterMarkInBytesOnTiers.get(tierAlias);
if (highWatermarkInBytes > reservedBytes
&& usedBytesOnTiers.get(tierAlias) >= highWatermarkInBytes) {
try {
mBlockWorker.freeSpace(Sessions.MIGRATE_DATA_SESSION_ID, reservedBytes, tierAlias);
} catch (WorkerOutOfSpaceException | BlockDoesNotExistException
| BlockAlreadyExistsException | InvalidWorkerStateException | IOException e) {
LOG.warn("SpaceReserver failed to free tier {} to {} bytes used",
tierAlias, reservedBytes, e.getMessage());
}
}
} else {
try {
mBlockWorker.freeSpace(Sessions.MIGRATE_DATA_SESSION_ID, reservedBytes, tierAlias);
} catch (WorkerOutOfSpaceException | BlockDoesNotExistException
| BlockAlreadyExistsException | InvalidWorkerStateException | IOException e) {
LOG.warn(e.getMessage());
}
}
}
}
Loading

0 comments on commit 457857f

Please sign in to comment.