Skip to content

Commit

Permalink
Refactor string deduplication and add spock (krzysztofslusarski#2)
Browse files Browse the repository at this point in the history 
* Refactor string deduplication and add spock
* fixing dedup parser - version update, bytes parser
* Add test to show no succeeding deduplication produces no stats

Co-authored-by: Artur Owczarek <[email protected]>
Co-authored-by: Ślusarski Krzysztof <[email protected]>
  • Loading branch information
@arturowczarek
3 people authored Dec 19, 2020
1 parent c206e1b commit ae69a82
Show file tree
Hide file tree
Showing 11 changed files with 294 additions and 59 deletions.
2 changes: 1 addition & 1 deletion gui-commons/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>parent</artifactId>
<artifactId>jvm-gc-logs-analyzer</artifactId>
<groupId>pl.ks.profiling</groupId>
<version>1.1-SNAPSHOT</version>
</parent>
Expand Down
2 changes: 1 addition & 1 deletion io-utils/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>parent</artifactId>
<artifactId>jvm-gc-logs-analyzer</artifactId>
<groupId>pl.ks.profiling</groupId>
<version>1.1-SNAPSHOT</version>
</parent>
Expand Down
45 changes: 44 additions & 1 deletion pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
<modelVersion>4.0.0</modelVersion>

<groupId>pl.ks.profiling</groupId>
<artifactId>parent</artifactId>
<artifactId>jvm-gc-logs-analyzer</artifactId>
<version>1.1-SNAPSHOT</version>
<modules>
<module>io-utils</module>
Expand Down Expand Up @@ -34,6 +34,8 @@
<commons-collections4.version>4.1</commons-collections4.version>
<xchart.version>3.6.0</xchart.version>
<miglayout-swing.version>5.0</miglayout-swing.version>
<spock.version>1.3-groovy-2.5</spock.version>
<gmavenplus-plugin.version>1.6.2</gmavenplus-plugin.version>
</properties>

<dependencyManagement>
Expand Down Expand Up @@ -78,6 +80,12 @@
<artifactId>miglayout-swing</artifactId>
<version>${miglayout-swing.version}</version>
</dependency>
<dependency>
<groupId>org.spockframework</groupId>
<artifactId>spock-core</artifactId>
<version>${spock.version}</version>
<scope>test</scope>
</dependency>
</dependencies>
</dependencyManagement>

Expand All @@ -91,6 +99,41 @@

<build>
<plugins>
<plugin>
<groupId>org.codehaus.gmavenplus</groupId>
<artifactId>gmavenplus-plugin</artifactId>
<version>${gmavenplus-plugin.version}</version>
<executions>
<execution>
<goals>
<goal>generateStubs</goal>
<goal>compile</goal>
<goal>compileTests</goal>
</goals>
</execution>
</executions>
</plugin>

<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>build-helper-maven-plugin</artifactId>
<executions>
<execution>
<id>add-groovy-test-source</id>
<phase>test</phase>
<goals>
<goal>add-test-source</goal>
</goals>
<configuration>
<sources>
<source>${basedir}/src/test/groovy</source>
<source>${basedir}/src/test/java</source>
</sources>
</configuration>
</execution>
</executions>
</plugin>

<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
Expand Down
6 changes: 5 additions & 1 deletion safepoint-analyzer-commons/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>parent</artifactId>
<artifactId>jvm-gc-logs-analyzer</artifactId>
<groupId>pl.ks.profiling</groupId>
<version>1.1-SNAPSHOT</version>
</parent>
Expand All @@ -29,5 +29,9 @@
<artifactId>gui-commons</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.spockframework</groupId>
<artifactId>spock-core</artifactId>
</dependency>
</dependencies>
</project>
6 changes: 6 additions & 0 deletions ...er-commons/src/main/java/pl/ks/profiling/safepoint/analyzer/commons/shared/Consumer3.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
package pl.ks.profiling.safepoint.analyzer.commons.shared;

@FunctionalInterface
public interface Consumer3<T, U, V> {
void apply(T t, U u, V v);
}
176 changes: 124 additions & 52 deletions ...filing/safepoint/analyzer/commons/shared/stringdedup/parser/StringDedupLogFileParser.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,74 +16,146 @@
package pl.ks.profiling.safepoint.analyzer.commons.shared.stringdedup.parser;

import java.math.BigDecimal;
import java.util.function.BiConsumer;
import pl.ks.profiling.safepoint.analyzer.commons.FileParser;
import pl.ks.profiling.safepoint.analyzer.commons.shared.Consumer3;
import pl.ks.profiling.safepoint.analyzer.commons.shared.ParserUtils;

public class StringDedupLogFileParser implements FileParser<StringDedupLogFile> {
public static final BigDecimal MB = new BigDecimal(1024);
private final StringDedupLogFile logFile = new StringDedupLogFile();

private final TokenParser[] parsers = new TokenParser[]{
new SimpleTokenParser("Concurrent String Deduplication", StringDedupLogFileParser::markDeduplicationStarted),
new SimpleTokenParser("Last Exec", StringDedupLogFileParser::markLastExecSection),
new SimpleTokenParser("Total Exec", StringDedupLogFileParser::markTotalExecOccurredSection),
new SizeCountParser("Deduplicated", StringDedupLogFile::deduplicated),
new SizeCountParser("Young", StringDedupLogFile::young),
new SizeCountParser("Old", StringDedupLogFile::old),
new SizeCountParser("New", StringDedupLogFile::newStrings, SizeCountParser.SIZE_MARKERS_WITHOUT_PARENTHESIS)
};

private static void markDeduplicationStarted(StringDedupLogFile logFile, String line) {
logFile.newEntry(ParserUtils.getTimeStamp(line));
}

private static void markLastExecSection(StringDedupLogFile logFile, String line) {
logFile.lastExecOccurred();
}

private static void markTotalExecOccurredSection(StringDedupLogFile logFile, String line) {
logFile.totalExecOccurred();
}

@Override
public void parseLine(String line) {
if (!line.contains("gc,stringdedup")) {
if (!isStringDeduplicationLine(line)) {
return;
}

if (line.contains("Concurrent String Deduplication")) {
logFile.newEntry(ParserUtils.getTimeStamp(line));
} else if (line.contains("Last Exec")) {
logFile.lastExecOccurred();
} else if (line.contains("Total Exec")) {
logFile.totalExecOccurred();
} else if (line.contains("Deduplicated")) {
int pos = line.indexOf("Deduplicated");
long count = ParserUtils.parseFirstNumber(line, pos);
int percentPos = line.indexOf("%)", pos);
BigDecimal size = ParserUtils.parseFirstBigDecimal(line, percentPos);
if (line.indexOf("M(", percentPos) >= 0) {
size = size.multiply(MB);
} else if (line.indexOf("G(", percentPos) >= 0) {
size = size.multiply(MB).multiply(MB);
}
logFile.deduplicated(count, size);
} else if (line.contains("Young")) {
int pos = line.indexOf("Young");
long count = ParserUtils.parseFirstNumber(line, pos);
int percentPos = line.indexOf("%)", pos);
BigDecimal size = ParserUtils.parseFirstBigDecimal(line, percentPos);
if (line.indexOf("M(", percentPos) >= 0) {
size = size.multiply(MB);
} else if (line.indexOf("G(", percentPos) >= 0) {
size = size.multiply(MB).multiply(MB);
useFirstAccepting(line, parsers);
}

@Override
public StringDedupLogFile fetchData() {
return logFile;
}

private void useFirstAccepting(String line, TokenParser[] parsers) {
for (TokenParser parser : parsers) {
if (parser.accepts(line)) {
parser.apply(logFile, line);
return;
}
logFile.young(count, size);
} else if (line.contains("Old")) {
int pos = line.indexOf("Old");
long count = ParserUtils.parseFirstNumber(line, pos);
int percentPos = line.indexOf("%)", pos);
BigDecimal size = ParserUtils.parseFirstBigDecimal(line, percentPos);
if (line.indexOf("M(", percentPos) >= 0) {
size = size.multiply(MB);
} else if (line.indexOf("G(", percentPos) >= 0) {
size = size.multiply(MB).multiply(MB);
}
}

private boolean isStringDeduplicationLine(String line) {
return line.contains("gc,stringdedup");
}

static abstract class TokenParser {
protected String token;

public TokenParser(String token) {
this.token = token;
}

public boolean accepts(String line) {
return line.contains(token);
}

abstract void apply(StringDedupLogFile logFile, String line);
}

static class SimpleTokenParser extends TokenParser {
private final BiConsumer<StringDedupLogFile, String> applyFunction;

public SimpleTokenParser(String token, BiConsumer<StringDedupLogFile, String> applyFunction) {
super(token);
this.applyFunction = applyFunction;
}

void apply(StringDedupLogFile logFile, String line) {
applyFunction.accept(logFile, line);
}
}

static class SizeCountParser extends TokenParser {
private final String bytesMarker;
private final String megaMarker;
private final String gigaMarker;
private final Consumer3<StringDedupLogFile, Long, BigDecimal> applyFunction;
private final static boolean SIZE_MARKERS_WITHOUT_PARENTHESIS = false;
public final static boolean SIZE_MARKERS_WITH_PARENTHESIS = true;

public SizeCountParser(String token, Consumer3<StringDedupLogFile, Long, BigDecimal> applyFunction) {
this(token, applyFunction, SIZE_MARKERS_WITH_PARENTHESIS);
}

public SizeCountParser(String token, Consumer3<StringDedupLogFile, Long, BigDecimal> applyFunction, boolean paraSizeMarker) {
super(token);
this.token = token;
this.applyFunction = applyFunction;

if (paraSizeMarker) {
this.bytesMarker = "B(";
this.megaMarker = "M(";
this.gigaMarker = "G(";
} else {
this.bytesMarker = "B";
this.megaMarker = "M";
this.gigaMarker = "G";
}
logFile.old(count, size);
} else if (line.contains("New")) {
int pos = line.indexOf("New");
long count = ParserUtils.parseFirstNumber(line, pos);
int percentPos = line.indexOf("%)", pos);
}

public void apply(StringDedupLogFile logFile, String line) {
int tokenPosition = line.indexOf(token);
long count = ParserUtils.parseFirstNumber(line, tokenPosition);
int percentPos = getPercentPos(line, tokenPosition);
BigDecimal sizeKb = getSizeInKiloBytes(line, percentPos);
applyFunction.apply(logFile, count, sizeKb);
}

private int getPercentPos(String line, int tokenPosition) {
return line.indexOf("%)", tokenPosition);
}

private BigDecimal getSizeInKiloBytes(String line, int percentPos) {
BigDecimal size = ParserUtils.parseFirstBigDecimal(line, percentPos);
if (line.indexOf("M", percentPos) >= 0) {
size = size.multiply(MB);
} else if (line.indexOf("G", percentPos) >= 0) {
size = size.multiply(MB).multiply(MB);
if (containsMarker(line, percentPos, bytesMarker)) {
return UnitsConverter.bytesToKiloBytes(size);
} else if (containsMarker(line, percentPos, megaMarker)) {
return UnitsConverter.megabytesToKiloBytes(size);
} else if (containsMarker(line, percentPos, gigaMarker)) {
return UnitsConverter.gigabytesToKiloBytes(size);
} else {
return size;
}
logFile.newStrings(count, size);
}
}

@Override
public StringDedupLogFile fetchData() {
return logFile;
private boolean containsMarker(String line, int percentPos, String marker) {
return line.indexOf(marker, percentPos) >= 0;
}
}

}
22 changes: 22 additions & 0 deletions .../pl/ks/profiling/safepoint/analyzer/commons/shared/stringdedup/parser/UnitsConverter.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
package pl.ks.profiling.safepoint.analyzer.commons.shared.stringdedup.parser;

import java.math.BigDecimal;
import java.math.RoundingMode;

class UnitsConverter {
public static final BigDecimal KB_TO_MB_MULTIPLIER = new BigDecimal(1024);
public static final BigDecimal KB_TO_GB_MULTIPLIER = KB_TO_MB_MULTIPLIER.multiply(KB_TO_MB_MULTIPLIER);
public static final BigDecimal B_TO_KB_DIVISOR = new BigDecimal(1024);

public static BigDecimal bytesToKiloBytes(BigDecimal bytes) {
return bytes.divide(B_TO_KB_DIVISOR, 2, RoundingMode.HALF_EVEN);
}

public static BigDecimal megabytesToKiloBytes(BigDecimal megaBytes) {
return megaBytes.multiply(KB_TO_MB_MULTIPLIER);
}

public static BigDecimal gigabytesToKiloBytes(BigDecimal gigaBytes) {
return gigaBytes.multiply(KB_TO_GB_MULTIPLIER);
}
}
Loading

0 comments on commit ae69a82

Please sign in to comment.