From 3c5291cde999422f0d5a2c97e4bda7afb3384bbb Mon Sep 17 00:00:00 2001 From: frensing Date: Mon, 24 Oct 2022 13:21:41 +0300 Subject: [PATCH 01/47] QuerySource --- .../cc/query/source/FileLineQuerySource.java | 37 +++++++++++++++++++ .../iguana/cc/query/source/QuerySource.java | 13 +++++++ 2 files changed, 50 insertions(+) create mode 100644 iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/FileLineQuerySource.java create mode 100644 iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/QuerySource.java diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/FileLineQuerySource.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/FileLineQuerySource.java new file mode 100644 index 000000000..fa72e2667 --- /dev/null +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/FileLineQuerySource.java @@ -0,0 +1,37 @@ +package org.aksw.iguana.cc.query.source; + +import org.aksw.iguana.cc.utils.FileUtils; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.util.List; + +public class FileLineQuerySource implements QuerySource { + + protected String path; + protected File queryFile; + + protected int size; + + public FileLineQuerySource(String path) throws IOException { + this.path = path; + this.queryFile = new File(this.path); + this.size = FileUtils.countLines(this.queryFile); + } + + @Override + public int size() { + return this.size; + } + + @Override + public String getQuery(int index) throws IOException { + return FileUtils.readLineAt(index, this.queryFile); + } + + @Override + public List getAllQueries() throws IOException { + return Files.readAllLines(this.queryFile.toPath()); + } +} diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/QuerySource.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/QuerySource.java new file mode 100644 index 000000000..365315ccb --- /dev/null +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/QuerySource.java @@ -0,0 +1,13 @@ +package org.aksw.iguana.cc.query.source; + +import java.io.IOException; +import java.util.List; + +public interface QuerySource { + + int size(); + + String getQuery(int index) throws IOException; + + List getAllQueries() throws IOException; +} From 3134afb31bf94caede06bc48c0c71dd15a2e28c2 Mon Sep 17 00:00:00 2001 From: frensing Date: Mon, 24 Oct 2022 14:14:46 +0300 Subject: [PATCH 02/47] QuerySet --- .../aksw/iguana/cc/query/set/QuerySet.java | 1 + .../query/set/newimpl/AbstractQuerySet.java | 25 +++++++++++ .../cc/query/set/newimpl/InMemQuerySet.java | 43 +++++++++++++++++++ 3 files changed, 69 insertions(+) create mode 100644 iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/newimpl/AbstractQuerySet.java create mode 100644 iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/newimpl/InMemQuerySet.java diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/QuerySet.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/QuerySet.java index b21073c37..be6e456ed 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/QuerySet.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/QuerySet.java @@ -19,5 +19,6 @@ public interface QuerySet { String getName(); + // TODO check if still needed String getContent() throws IOException; } diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/newimpl/AbstractQuerySet.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/newimpl/AbstractQuerySet.java new file mode 100644 index 000000000..7ae890fb9 --- /dev/null +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/newimpl/AbstractQuerySet.java @@ -0,0 +1,25 @@ +package org.aksw.iguana.cc.query.set.newimpl; + +import org.aksw.iguana.cc.query.set.QuerySet; +import org.aksw.iguana.cc.query.source.QuerySource; + +public abstract class AbstractQuerySet implements QuerySet { + protected QuerySource querySource; + + protected String name; + + public AbstractQuerySet(String name, QuerySource querySource) { + this.name = name; + this.querySource = querySource; + } + + @Override + public int size() { + return this.querySource.size(); + } + + @Override + public String getName() { + return this.name; + } +} diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/newimpl/InMemQuerySet.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/newimpl/InMemQuerySet.java new file mode 100644 index 000000000..d1d01584f --- /dev/null +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/newimpl/InMemQuerySet.java @@ -0,0 +1,43 @@ +package org.aksw.iguana.cc.query.set.newimpl; + +import org.aksw.iguana.cc.query.source.QuerySource; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.List; + +public class InMemQuerySet extends AbstractQuerySet { + + private static final Logger LOGGER = LoggerFactory.getLogger(InMemQuerySet.class); + + private List queries; + + public InMemQuerySet(String name, QuerySource querySource) { + super(name, querySource); + loadQueries(); + } + + private void loadQueries() { + try { + this.queries = this.querySource.getAllQueries(); + } catch (IOException e) { + LOGGER.error("Could not read queries"); + } + } + + @Override + public String getQueryAtPos(int pos) throws IOException { + return this.queries.get(pos); + } + + @Override + public int size() { + return this.queries.size(); + } + + @Override + public String getContent() throws IOException { + return null; + } +} From 2b9266fa296295fa71f75503829ca2fbd28e5f35 Mon Sep 17 00:00:00 2001 From: frensing Date: Mon, 24 Oct 2022 14:31:19 +0300 Subject: [PATCH 03/47] QuerySelector --- .../query/selector/AbstractQuerySelector.java | 10 +++++++++ .../query/selector/LinearQuerySelector.java | 20 ++++++++++++++++++ .../cc/query/selector/QuerySelector.java | 5 +++++ .../query/selector/QuerySelectorFactory.java | 6 ++++++ .../query/selector/RandomQuerySelector.java | 21 +++++++++++++++++++ 5 files changed, 62 insertions(+) create mode 100644 iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/selector/AbstractQuerySelector.java create mode 100644 iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/selector/LinearQuerySelector.java create mode 100644 iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/selector/QuerySelector.java create mode 100644 iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/selector/QuerySelectorFactory.java create mode 100644 iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/selector/RandomQuerySelector.java diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/selector/AbstractQuerySelector.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/selector/AbstractQuerySelector.java new file mode 100644 index 000000000..b3d77d51b --- /dev/null +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/selector/AbstractQuerySelector.java @@ -0,0 +1,10 @@ +package org.aksw.iguana.cc.query.selector; + +public abstract class AbstractQuerySelector implements QuerySelector { + + protected int size; + + public AbstractQuerySelector(int size) { + this.size = size; + } +} diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/selector/LinearQuerySelector.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/selector/LinearQuerySelector.java new file mode 100644 index 000000000..9b0af6bf1 --- /dev/null +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/selector/LinearQuerySelector.java @@ -0,0 +1,20 @@ +package org.aksw.iguana.cc.query.selector; + +import org.aksw.iguana.commons.annotation.Shorthand; + +@Shorthand("LinearQuerySelector") +public class LinearQuerySelector extends AbstractQuerySelector { + protected int querySelector; + + public LinearQuerySelector(int size) { + super(size); + } + + @Override + public int getNextQuery() { + if (this.querySelector >= this.size) { + this.querySelector = 0; + } + return this.querySelector++; + } +} diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/selector/QuerySelector.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/selector/QuerySelector.java new file mode 100644 index 000000000..0cd4c035d --- /dev/null +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/selector/QuerySelector.java @@ -0,0 +1,5 @@ +package org.aksw.iguana.cc.query.selector; + +public interface QuerySelector { + int getNextQuery(); +} diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/selector/QuerySelectorFactory.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/selector/QuerySelectorFactory.java new file mode 100644 index 000000000..968b40a77 --- /dev/null +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/selector/QuerySelectorFactory.java @@ -0,0 +1,6 @@ +package org.aksw.iguana.cc.query.selector; + +import org.aksw.iguana.commons.factory.TypedFactory; + +public class QuerySelectorFactory extends TypedFactory { +} diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/selector/RandomQuerySelector.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/selector/RandomQuerySelector.java new file mode 100644 index 000000000..fdb398ebc --- /dev/null +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/selector/RandomQuerySelector.java @@ -0,0 +1,21 @@ +package org.aksw.iguana.cc.query.selector; + +import org.aksw.iguana.commons.annotation.Shorthand; + +import java.util.Random; + +@Shorthand("RandomQuerySelector") +public class RandomQuerySelector extends AbstractQuerySelector { + + protected Random querySelector; + + public RandomQuerySelector(int size, long seed) { + super(size); + this.querySelector = new Random(seed); + } + + @Override + public int getNextQuery() { + return this.querySelector.nextInt(this.size); + } +} From c534cd3abb9be48f602eb18a3d3930b06a122652 Mon Sep 17 00:00:00 2001 From: frensing Date: Mon, 24 Oct 2022 17:16:31 +0300 Subject: [PATCH 04/47] FileSeparatorQuerySource --- .../source/FileSeparatorQuerySource.java | 107 ++++++++++++++++++ .../org/aksw/iguana/cc/utils/FileUtils.java | 21 ++-- .../source/FileSeparatorQuerySourceTest.java | 57 ++++++++++ .../source/separated-queries-default.txt | 11 ++ .../query/source/separated-queries-space.txt | 11 ++ 5 files changed, 197 insertions(+), 10 deletions(-) create mode 100644 iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/FileSeparatorQuerySource.java create mode 100644 iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/source/FileSeparatorQuerySourceTest.java create mode 100644 iguana.corecontroller/src/test/resources/query/source/separated-queries-default.txt create mode 100644 iguana.corecontroller/src/test/resources/query/source/separated-queries-space.txt diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/FileSeparatorQuerySource.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/FileSeparatorQuerySource.java new file mode 100644 index 000000000..66f1023e3 --- /dev/null +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/FileSeparatorQuerySource.java @@ -0,0 +1,107 @@ +package org.aksw.iguana.cc.query.source; + +import org.aksw.iguana.cc.utils.FileUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.List; +import java.util.stream.Stream; + +public class FileSeparatorQuerySource implements QuerySource { + private static final Logger LOGGER = LoggerFactory.getLogger(FileSeparatorQuerySource.class); + + private static final String DEFAULT_SEPARATOR = "###"; + + protected String path; + protected File queryFile; + protected String separator; + protected int size; + + private List separatorPositions; + + public FileSeparatorQuerySource(String path) { + this(path, DEFAULT_SEPARATOR); + } + + public FileSeparatorQuerySource(String path, String separator) { + this.path = path; + this.queryFile = new File(this.path); + this.separator = separator; + + indexFile(); + } + + private void indexFile() { + this.separatorPositions = new LinkedList<>(); + int separatorCount = 0; + try { + BufferedReader reader = FileUtils.getBufferedReader(this.queryFile); + int index = 0; + String line; + this.separatorPositions.add(-1); + while ((line = reader.readLine()) != null) { + if (line.equals(this.separator)) { + separatorCount++; + this.separatorPositions.add(index); + } + index++; + } + this.separatorPositions.add(index); + reader.close(); + } catch (IOException e) { + LOGGER.error("Could not read queries"); + } + + this.size = separatorCount + 1; + } + + @Override + public int size() { + return this.size; + } + + @Override + public String getQuery(int index) throws IOException { + int start = this.separatorPositions.get(index) + 1; + int end = this.separatorPositions.get(index + 1); + + try (Stream lines = Files.lines(this.queryFile.toPath())) { + return lines.skip(start) + .limit(end - start) + .reduce((a, b) -> a + b) + .get(); + } catch (FileNotFoundException e) { + LOGGER.error("Could not read queries"); + } + + // todo + return null; + } + + @Override + public List getAllQueries() throws IOException { + try (BufferedReader reader = FileUtils.getBufferedReader(this.queryFile)) { + List queries = new ArrayList<>(this.size); + String line; + StringBuilder query = new StringBuilder(); + while ((line = reader.readLine()) != null) { + if (line.equals(this.separator)) { + queries.add(query.toString()); + query = new StringBuilder(); + } else { + query.append(line); + } + } + queries.add(query.toString()); + return queries; + } + } + +} diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/utils/FileUtils.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/utils/FileUtils.java index 69c2a9a94..3ba6269a6 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/utils/FileUtils.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/utils/FileUtils.java @@ -15,19 +15,18 @@ public class FileUtils { /** * Counts the lines in a file efficently Props goes to: - * http://stackoverflow.com/a/453067/2917596 - * - * + * ... + * * @param filename File to count lines of * @return No. of lines in File * @throws IOException */ public static int countLines(File filename) throws IOException { - try (InputStream is = new BufferedInputStream(new FileInputStream(filename));) { + try (InputStream is = new BufferedInputStream(new FileInputStream(filename))) { byte[] c = new byte[1024]; int count = 0; - int readChars = 0; + int readChars; boolean empty = true; byte lastChar = '\n'; while ((readChars = is.read(c)) != -1) { @@ -53,19 +52,18 @@ public static int countLines(File filename) throws IOException { /** * Returns a line at a given position of a File * - * * @param pos line which should be returned * @param filename File in which the queries are stated * @return line at pos * @throws IOException */ public static String readLineAt(int pos, File filename) throws IOException { - try (InputStream is = new BufferedInputStream(new FileInputStream(filename));){ - StringBuilder line = new StringBuilder(); - + try (InputStream is = new BufferedInputStream(new FileInputStream(filename))) { + StringBuilder line = new StringBuilder(); + byte[] c = new byte[1024]; int count = 0; - int readChars = 0; + int readChars; byte lastChar = '\n'; while ((readChars = is.read(c)) != -1) { for (int i = 0; i < readChars; ++i) { @@ -102,4 +100,7 @@ public static String readFile(String path) throws IOException { return new String(encoded, StandardCharsets.UTF_8); } + public static BufferedReader getBufferedReader(File queryFile) throws FileNotFoundException { + return new BufferedReader(new FileReader(queryFile)); + } } diff --git a/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/source/FileSeparatorQuerySourceTest.java b/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/source/FileSeparatorQuerySourceTest.java new file mode 100644 index 000000000..2a16ccaa7 --- /dev/null +++ b/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/source/FileSeparatorQuerySourceTest.java @@ -0,0 +1,57 @@ +package org.aksw.iguana.cc.query.source; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +import static org.junit.Assert.assertEquals; + +@RunWith(Parameterized.class) +public class FileSeparatorQuerySourceTest { + + private final FileSeparatorQuerySource querySource; + + public FileSeparatorQuerySourceTest(String path, String separator) { + if (separator == null) { + this.querySource = new FileSeparatorQuerySource(path); + } else { + this.querySource = new FileSeparatorQuerySource(path, separator); + } + } + + @Parameterized.Parameters + public static Collection data() { + Collection testData = new ArrayList<>(); + testData.add(new Object[]{"src/test/resources/query/source/separated-queries-default.txt", null}); + testData.add(new Object[]{"src/test/resources/query/source/separated-queries-space.txt", ""}); + + return testData; + } + + @Test + public void size() { + assertEquals(3, this.querySource.size()); + } + + @Test + public void getQuery() throws IOException { + assertEquals("QUERY 1 {still query 1}", this.querySource.getQuery(0)); + assertEquals("QUERY 2 {still query 2}", this.querySource.getQuery(1)); + assertEquals("QUERY 3 {still query 3}", this.querySource.getQuery(2)); + } + + @Test + public void getAllQueries() throws IOException { + List expected = new ArrayList<>(3); + expected.add("QUERY 1 {still query 1}"); + expected.add("QUERY 2 {still query 2}"); + expected.add("QUERY 3 {still query 3}"); + + assertEquals(expected, this.querySource.getAllQueries()); + } +} \ No newline at end of file diff --git a/iguana.corecontroller/src/test/resources/query/source/separated-queries-default.txt b/iguana.corecontroller/src/test/resources/query/source/separated-queries-default.txt new file mode 100644 index 000000000..0147adaa6 --- /dev/null +++ b/iguana.corecontroller/src/test/resources/query/source/separated-queries-default.txt @@ -0,0 +1,11 @@ +QUERY 1 { +still query 1 +} +### +QUERY 2 { +still query 2 +} +### +QUERY 3 { +still query 3 +} \ No newline at end of file diff --git a/iguana.corecontroller/src/test/resources/query/source/separated-queries-space.txt b/iguana.corecontroller/src/test/resources/query/source/separated-queries-space.txt new file mode 100644 index 000000000..9b948028b --- /dev/null +++ b/iguana.corecontroller/src/test/resources/query/source/separated-queries-space.txt @@ -0,0 +1,11 @@ +QUERY 1 { +still query 1 +} + +QUERY 2 { +still query 2 +} + +QUERY 3 { +still query 3 +} \ No newline at end of file From f829e418c6b662c9b3cb23dbe6dfe7e27acbeaed Mon Sep 17 00:00:00 2001 From: frensing Date: Mon, 24 Oct 2022 17:30:27 +0300 Subject: [PATCH 05/47] FolderQuerySource --- .../cc/query/source/FolderQuerySource.java | 66 +++++++++++++++++++ .../query/source/FolderQuerySourceTest.java | 40 +++++++++++ .../query/source/query-folder/query1.txt | 3 + .../query/source/query-folder/query2.txt | 3 + 4 files changed, 112 insertions(+) create mode 100644 iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/FolderQuerySource.java create mode 100644 iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/source/FolderQuerySourceTest.java create mode 100644 iguana.corecontroller/src/test/resources/query/source/query-folder/query1.txt create mode 100644 iguana.corecontroller/src/test/resources/query/source/query-folder/query2.txt diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/FolderQuerySource.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/FolderQuerySource.java new file mode 100644 index 000000000..1c9e65ba2 --- /dev/null +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/FolderQuerySource.java @@ -0,0 +1,66 @@ +package org.aksw.iguana.cc.query.source; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +public class FolderQuerySource implements QuerySource { + + protected static final Logger LOGGER = LoggerFactory.getLogger(FolderQuerySource.class); + + protected String path; + protected File[] files; + + public FolderQuerySource(String path) { + this.path = path; + + indexFolder(); + } + + private void indexFolder() { + File dir = new File(this.path); + if (!dir.exists()) { + LOGGER.error("Folder does not exist"); + return; + } + if (!dir.isDirectory()) { + LOGGER.error("Path is not a folder"); + return; + } + + LOGGER.info("indexing folder {}", this.path); + this.files = dir.listFiles(); + } + + @Override + public int size() { + return this.files.length; + } + + @Override + public String getQuery(int index) throws IOException { + try (BufferedReader reader = new BufferedReader(new FileReader(this.files[index]))) { + StringBuilder query = new StringBuilder(); + String line; + while ((line = reader.readLine()) != null) { + query.append(line); + } + return query.toString(); + } + } + + @Override + public List getAllQueries() throws IOException { + List queries = new ArrayList<>(this.files.length); + for (int i = 0; i < this.files.length; i++) { + queries.add(getQuery(i)); + } + return queries; + } +} diff --git a/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/source/FolderQuerySourceTest.java b/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/source/FolderQuerySourceTest.java new file mode 100644 index 000000000..ee87a55c5 --- /dev/null +++ b/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/source/FolderQuerySourceTest.java @@ -0,0 +1,40 @@ +package org.aksw.iguana.cc.query.source; + +import org.junit.Test; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.assertEquals; + +public class FolderQuerySourceTest { + + private static final String PATH = "src/test/resources/query/source/query-folder"; + + private final FolderQuerySource querySource; + + public FolderQuerySourceTest() { + this.querySource = new FolderQuerySource(PATH); + } + + @Test + public void size() { + assertEquals(2, this.querySource.size()); + } + + @Test + public void getQuery() throws IOException { + assertEquals("QUERY 1 {still query 1}", this.querySource.getQuery(0)); + assertEquals("QUERY 2 {still query 2}", this.querySource.getQuery(1)); + } + + @Test + public void getAllQueries() throws IOException { + List expected = new ArrayList<>(3); + expected.add("QUERY 1 {still query 1}"); + expected.add("QUERY 2 {still query 2}"); + + assertEquals(expected, this.querySource.getAllQueries()); + } +} \ No newline at end of file diff --git a/iguana.corecontroller/src/test/resources/query/source/query-folder/query1.txt b/iguana.corecontroller/src/test/resources/query/source/query-folder/query1.txt new file mode 100644 index 000000000..fdef9bb9d --- /dev/null +++ b/iguana.corecontroller/src/test/resources/query/source/query-folder/query1.txt @@ -0,0 +1,3 @@ +QUERY 1 { +still query 1 +} \ No newline at end of file diff --git a/iguana.corecontroller/src/test/resources/query/source/query-folder/query2.txt b/iguana.corecontroller/src/test/resources/query/source/query-folder/query2.txt new file mode 100644 index 000000000..976f82c51 --- /dev/null +++ b/iguana.corecontroller/src/test/resources/query/source/query-folder/query2.txt @@ -0,0 +1,3 @@ +QUERY 2 { +still query 2 +} \ No newline at end of file From f533fea27ea0a7c038653bacdd224397058686e8 Mon Sep 17 00:00:00 2001 From: frensing Date: Mon, 24 Oct 2022 17:36:13 +0300 Subject: [PATCH 06/47] FileLineQuerySourceTest --- .../query/source/FileLineQuerySourceTest.java | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/source/FileLineQuerySourceTest.java diff --git a/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/source/FileLineQuerySourceTest.java b/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/source/FileLineQuerySourceTest.java new file mode 100644 index 000000000..be142841d --- /dev/null +++ b/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/source/FileLineQuerySourceTest.java @@ -0,0 +1,42 @@ +package org.aksw.iguana.cc.query.source; + +import org.junit.Test; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.assertEquals; + +public class FileLineQuerySourceTest { + + private static final String PATH = "src/test/resources/query/source/queries.txt"; + + private final FileLineQuerySource querySource; + + public FileLineQuerySourceTest() throws IOException { + this.querySource = new FileLineQuerySource(PATH); + } + + @Test + public void size() { + assertEquals(3, this.querySource.size()); + } + + @Test + public void getQuery() throws IOException { + assertEquals("QUERY 1 {still query 1}", this.querySource.getQuery(0)); + assertEquals("QUERY 2 {still query 2}", this.querySource.getQuery(1)); + assertEquals("QUERY 3 {still query 3}", this.querySource.getQuery(2)); + } + + @Test + public void getAllQueries() throws IOException { + List expected = new ArrayList<>(3); + expected.add("QUERY 1 {still query 1}"); + expected.add("QUERY 2 {still query 2}"); + expected.add("QUERY 3 {still query 3}"); + + assertEquals(expected, this.querySource.getAllQueries()); + } +} \ No newline at end of file From 7ee76ec6e5b4010abc83529f86616f22157c1b5f Mon Sep 17 00:00:00 2001 From: frensing Date: Mon, 24 Oct 2022 17:47:30 +0300 Subject: [PATCH 07/47] remove getContent from QuerySet --- .../aksw/iguana/cc/query/set/QuerySet.java | 3 --- .../cc/query/set/impl/FileBasedQuerySet.java | 20 ++++++------------- .../cc/query/set/impl/InMemQuerySet.java | 5 ----- .../query/set/newimpl/FileBasedQuerySet.java | 17 ++++++++++++++++ .../cc/query/set/newimpl/InMemQuerySet.java | 5 ----- .../iguana/cc/worker/impl/UPDATEWorker.java | 6 +++--- .../test/resources/query/source/queries.txt | 3 +++ 7 files changed, 29 insertions(+), 30 deletions(-) create mode 100644 iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/newimpl/FileBasedQuerySet.java create mode 100644 iguana.corecontroller/src/test/resources/query/source/queries.txt diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/QuerySet.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/QuerySet.java index be6e456ed..cbae48a6d 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/QuerySet.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/QuerySet.java @@ -18,7 +18,4 @@ public interface QuerySet { int size(); String getName(); - - // TODO check if still needed - String getContent() throws IOException; } diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/impl/FileBasedQuerySet.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/impl/FileBasedQuerySet.java index 54093b17f..1adace21e 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/impl/FileBasedQuerySet.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/impl/FileBasedQuerySet.java @@ -11,17 +11,16 @@ */ public class FileBasedQuerySet implements QuerySet { - private File queryFile; - private int size=0; + private final File queryFile; + private int size; - - public FileBasedQuerySet(File queryFile) throws IOException { - this.queryFile=queryFile; - size=FileUtils.countLines(queryFile); + public FileBasedQuerySet(File queryFile) throws IOException { + this.queryFile = queryFile; + size = FileUtils.countLines(queryFile); } - public File getFile(){ + public File getFile() { return queryFile; } @@ -39,11 +38,4 @@ public int size() { public String getName() { return queryFile.getName(); } - - @Override - public String getContent() throws IOException { - return org.apache.commons.io.FileUtils.readFileToString(queryFile, "UTF-8"); - } - - } diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/impl/InMemQuerySet.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/impl/InMemQuerySet.java index ade10b206..8fd63b78a 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/impl/InMemQuerySet.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/impl/InMemQuerySet.java @@ -29,9 +29,4 @@ public int size() { public String getName() { return name; } - - @Override - public String getContent() throws IOException { - return queries.toString(); - } } diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/newimpl/FileBasedQuerySet.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/newimpl/FileBasedQuerySet.java new file mode 100644 index 000000000..8bd276e59 --- /dev/null +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/newimpl/FileBasedQuerySet.java @@ -0,0 +1,17 @@ +package org.aksw.iguana.cc.query.set.newimpl; + +import org.aksw.iguana.cc.query.source.QuerySource; + +import java.io.IOException; + +public class FileBasedQuerySet extends AbstractQuerySet { + + public FileBasedQuerySet(String name, QuerySource querySource) { + super(name, querySource); + } + + @Override + public String getQueryAtPos(int pos) throws IOException { + return this.querySource.getQuery(pos); + } +} diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/newimpl/InMemQuerySet.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/newimpl/InMemQuerySet.java index d1d01584f..a29c3379b 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/newimpl/InMemQuerySet.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/newimpl/InMemQuerySet.java @@ -35,9 +35,4 @@ public String getQueryAtPos(int pos) throws IOException { public int size() { return this.queries.size(); } - - @Override - public String getContent() throws IOException { - return null; - } } diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/impl/UPDATEWorker.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/impl/UPDATEWorker.java index befd00ebc..f469712d9 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/impl/UPDATEWorker.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/impl/UPDATEWorker.java @@ -26,7 +26,7 @@ public class UPDATEWorker extends HttpPostWorker { private int currentQueryID = 0; private UpdateTimer updateTimer = new UpdateTimer(); - private String timerStrategy; + private final String timerStrategy; public UPDATEWorker(String taskID, Connection connection, String queriesFile, @Nullable String timerStrategy, @Nullable Integer timeOut, @Nullable Integer timeLimit, @Nullable Integer fixedLatency, @Nullable Integer gaussianLatency, Integer workerID) { super(taskID, connection, queriesFile, "application/sparql-update", null, null, "lang.SPARQL", timeOut, timeLimit, fixedLatency, gaussianLatency, workerID); @@ -84,7 +84,7 @@ public void getNextQuery(StringBuilder queryStr, StringBuilder queryID) throws I QuerySet currentQueryFile = this.queryFileList[this.currentQueryID++]; queryID.append(currentQueryFile.getName()); - queryStr.append(currentQueryFile.getContent()); + queryStr.append(currentQueryFile.getQueryAtPos(0)); } @@ -114,7 +114,7 @@ private void setUpdateTimer(String strategyStr) { break; case DISTRIBUTED: if (timeLimit != null) { - this.updateTimer = new UpdateTimer(this.queryFileList.length, (double) this.timeLimit); + this.updateTimer = new UpdateTimer(this.queryFileList.length, this.timeLimit); } else { LOGGER.warn("Worker[{{}} : {{}}]: DISTRIBUTED Updates can only be used with timeLimit!", workerType, workerID); diff --git a/iguana.corecontroller/src/test/resources/query/source/queries.txt b/iguana.corecontroller/src/test/resources/query/source/queries.txt new file mode 100644 index 000000000..c62f4a847 --- /dev/null +++ b/iguana.corecontroller/src/test/resources/query/source/queries.txt @@ -0,0 +1,3 @@ +QUERY 1 {still query 1} +QUERY 2 {still query 2} +QUERY 3 {still query 3} \ No newline at end of file From 746c2ec57f7b5042b88b8cb3ce578c1c264e5030 Mon Sep 17 00:00:00 2001 From: frensing Date: Mon, 24 Oct 2022 17:55:08 +0300 Subject: [PATCH 08/47] QuerySelector and LinearQuerySelectorTest --- .../query/selector/LinearQuerySelector.java | 2 +- .../cc/query/selector/QuerySelector.java | 2 +- .../query/selector/QuerySelectorFactory.java | 6 ----- .../query/selector/RandomQuerySelector.java | 2 +- .../selector/LinearQuerySelectorTest.java | 23 +++++++++++++++++++ 5 files changed, 26 insertions(+), 9 deletions(-) delete mode 100644 iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/selector/QuerySelectorFactory.java create mode 100644 iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/selector/LinearQuerySelectorTest.java diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/selector/LinearQuerySelector.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/selector/LinearQuerySelector.java index 9b0af6bf1..52d089a37 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/selector/LinearQuerySelector.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/selector/LinearQuerySelector.java @@ -11,7 +11,7 @@ public LinearQuerySelector(int size) { } @Override - public int getNextQuery() { + public int getNextIndex() { if (this.querySelector >= this.size) { this.querySelector = 0; } diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/selector/QuerySelector.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/selector/QuerySelector.java index 0cd4c035d..fb08a1700 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/selector/QuerySelector.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/selector/QuerySelector.java @@ -1,5 +1,5 @@ package org.aksw.iguana.cc.query.selector; public interface QuerySelector { - int getNextQuery(); + int getNextIndex(); } diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/selector/QuerySelectorFactory.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/selector/QuerySelectorFactory.java deleted file mode 100644 index 968b40a77..000000000 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/selector/QuerySelectorFactory.java +++ /dev/null @@ -1,6 +0,0 @@ -package org.aksw.iguana.cc.query.selector; - -import org.aksw.iguana.commons.factory.TypedFactory; - -public class QuerySelectorFactory extends TypedFactory { -} diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/selector/RandomQuerySelector.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/selector/RandomQuerySelector.java index fdb398ebc..fda40effc 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/selector/RandomQuerySelector.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/selector/RandomQuerySelector.java @@ -15,7 +15,7 @@ public RandomQuerySelector(int size, long seed) { } @Override - public int getNextQuery() { + public int getNextIndex() { return this.querySelector.nextInt(this.size); } } diff --git a/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/selector/LinearQuerySelectorTest.java b/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/selector/LinearQuerySelectorTest.java new file mode 100644 index 000000000..52e5e4cfd --- /dev/null +++ b/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/selector/LinearQuerySelectorTest.java @@ -0,0 +1,23 @@ +package org.aksw.iguana.cc.query.selector; + +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +public class LinearQuerySelectorTest { + + private LinearQuerySelector linearQuerySelector; + + @Before + public void setUp() { + this.linearQuerySelector = new LinearQuerySelector(5); + } + + @Test + public void getNextIndex() { + for (int i = 0; i < 10; i++) { + assertEquals(i % 5, this.linearQuerySelector.getNextIndex()); + } + } +} \ No newline at end of file From 315df0630ce0354c845348eead2082f93ef2fb08 Mon Sep 17 00:00:00 2001 From: frensing Date: Mon, 24 Oct 2022 18:34:41 +0300 Subject: [PATCH 09/47] QueryHandler --- .../iguana/cc/query/handler/QueryHandler.java | 134 ++++++++++++++++++ .../cc/query/handler/QueryHandlerTest.java | 87 ++++++++++++ 2 files changed, 221 insertions(+) create mode 100644 iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/handler/QueryHandler.java create mode 100644 iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/handler/QueryHandlerTest.java diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/handler/QueryHandler.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/handler/QueryHandler.java new file mode 100644 index 000000000..585e63fee --- /dev/null +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/handler/QueryHandler.java @@ -0,0 +1,134 @@ +package org.aksw.iguana.cc.query.handler; + +import org.aksw.iguana.cc.query.selector.LinearQuerySelector; +import org.aksw.iguana.cc.query.selector.QuerySelector; +import org.aksw.iguana.cc.query.selector.RandomQuerySelector; +import org.aksw.iguana.cc.query.set.QuerySet; +import org.aksw.iguana.cc.query.set.newimpl.FileBasedQuerySet; +import org.aksw.iguana.cc.query.set.newimpl.InMemQuerySet; +import org.aksw.iguana.cc.query.source.FileLineQuerySource; +import org.aksw.iguana.cc.query.source.FileSeparatorQuerySource; +import org.aksw.iguana.cc.query.source.FolderQuerySource; +import org.aksw.iguana.cc.query.source.QuerySource; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.Map; + +public class QueryHandler { + + protected final Logger LOGGER = LoggerFactory.getLogger(QueryHandler.class); + + protected Map config; + protected Integer workerID; + protected String location; + + protected boolean caching; + + protected QuerySelector querySelector; + + protected QuerySet querySet; + + public QueryHandler(Map config, Integer workerID) { + this.config = config; + this.workerID = workerID; + + this.location = (String) config.get("location"); + + initQuerySet(); + initQuerySelector(); + + // TODO pattern + } + + public void getNextQuery(StringBuilder queryStr, StringBuilder queryID) throws IOException { + int queryIndex = this.querySelector.getNextIndex(); + queryStr.append(this.querySet.getQueryAtPos(queryIndex)); + queryID.append(this.querySet.getName()).append(":").append(queryIndex); + } + + private void initQuerySet() { + try { + QuerySource querySource = getQuerySource(); + + Boolean caching = (Boolean) config.get("caching"); + if (caching == null) { + caching = true; + } + this.caching = caching; + + if (this.caching) { + this.querySet = new InMemQuerySet(this.location, querySource); + } else { + this.querySet = new FileBasedQuerySet(this.location, querySource); + } + } catch (IOException e) { + LOGGER.error("Could not create QuerySource", e); + } + } + + private QuerySource getQuerySource() throws IOException { + Object formatObj = this.config.get("format"); + if (formatObj == null) { // Default + return new FileLineQuerySource(this.location); + } + + if (formatObj instanceof String) { + String f = (String) formatObj; + if (f.equals("one_per_line")) { + return new FileLineQuerySource(this.location); + } + if (f.equals("separator")) { // No custom separator given -> use "###" + return new FileSeparatorQuerySource(this.location); + } + + LOGGER.error("Unknown query format: {}", f); + } + + if (formatObj instanceof Map) { + Map format = (Map) formatObj; + if (format.containsKey("separator")) { + String separator = (String) format.get("separator"); + return new FileSeparatorQuerySource(this.location, separator); + } + if (format.containsKey("folder")) { + return new FolderQuerySource(this.location); + } + + LOGGER.error("Unknown query format: {}", format); + } + + return null; + } + + private void initQuerySelector() { + Object orderObj = this.config.get("order"); + if (orderObj == null) { // Default + this.querySelector = new LinearQuerySelector(this.querySet.size()); + } + + if (orderObj instanceof String) { + String order = (String) orderObj; + if (order.equals("linear")) { + this.querySelector = new LinearQuerySelector(this.querySet.size()); + return; + } + if (order.equals("random")) { + this.querySelector = new RandomQuerySelector(this.querySet.size(), this.workerID); + return; + } + + LOGGER.error("Unknown order: " + order); + } + if (orderObj instanceof Map) { + Map order = (Map) orderObj; + if (order.containsKey("seed")) { + Integer seed = (Integer) order.get("seed"); + this.querySelector = new RandomQuerySelector(this.querySet.size(), seed); + return; + } + LOGGER.error("Unknown order: " + order); + } + } +} diff --git a/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/handler/QueryHandlerTest.java b/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/handler/QueryHandlerTest.java new file mode 100644 index 000000000..78008464d --- /dev/null +++ b/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/handler/QueryHandlerTest.java @@ -0,0 +1,87 @@ +package org.aksw.iguana.cc.query.handler; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; + +import static org.junit.Assert.assertEquals; + +@RunWith(Parameterized.class) +public class QueryHandlerTest { + + private final QueryHandler queryHandler; + private final String[] expected; + + public QueryHandlerTest(Map config, String[] expected) { + this.queryHandler = new QueryHandler(config, 0); // workerID 0 results in correct seed for RandomSelector + this.expected = expected; + } + + @Parameterized.Parameters + public static Collection data() { + String[] linear = new String[]{"QUERY 1 {still query 1}", "QUERY 2 {still query 2}", "QUERY 3 {still query 3}", "QUERY 1 {still query 1}"}; + String[] random = new String[]{"QUERY 1 {still query 1}", "QUERY 2 {still query 2}", "QUERY 2 {still query 2}", "QUERY 3 {still query 3}"}; + + Collection testData = new ArrayList<>(); + + // Defaults: one-per-line, caching, linear + Map config1 = new HashMap<>(); + config1.put("location", "src/test/resources/query/source/queries.txt"); + testData.add(new Object[]{config1, linear}); + + // Defaults: separator("###"), caching, linear + Map config2 = new HashMap<>(); + config2.put("location", "src/test/resources/query/source/separated-queries-default.txt"); + config2.put("format", "separator"); + testData.add(new Object[]{config2, linear}); + + + Map config3 = new HashMap<>(); + config3.put("location", "src/test/resources/query/source/separated-queries-default.txt"); + Map format3 = new HashMap<>(); + format3.put("separator", "###"); + config3.put("format", format3); + config3.put("caching", false); + config3.put("order", "random"); + testData.add(new Object[]{config3, random}); + + // Defaults: one-per-line, caching + Map config4 = new HashMap<>(); + config4.put("location", "src/test/resources/query/source/queries.txt"); + Map order4 = new HashMap<>(); + order4.put("seed", 0); + config4.put("order", order4); + testData.add(new Object[]{config4, random}); + + return testData; + } + + @Test + public void getNextQuery() throws IOException { + StringBuilder query = new StringBuilder(); + StringBuilder queryID = new StringBuilder(); + this.queryHandler.getNextQuery(query, queryID); + assertEquals(this.expected[0], query.toString()); + + query = new StringBuilder(); + queryID = new StringBuilder(); + this.queryHandler.getNextQuery(query, queryID); + assertEquals(this.expected[1], query.toString()); + + query = new StringBuilder(); + queryID = new StringBuilder(); + this.queryHandler.getNextQuery(query, queryID); + assertEquals(this.expected[2], query.toString()); + + query = new StringBuilder(); + queryID = new StringBuilder(); + this.queryHandler.getNextQuery(query, queryID); + assertEquals(this.expected[3], query.toString()); + } +} \ No newline at end of file From 884cd81c3a8f543fa0eb8bbe5fa05cfd1444f82a Mon Sep 17 00:00:00 2001 From: frensing Date: Mon, 24 Oct 2022 18:45:28 +0300 Subject: [PATCH 10/47] QueryHandler add folder test --- .../org/aksw/iguana/cc/query/handler/QueryHandler.java | 6 +++--- .../aksw/iguana/cc/query/handler/QueryHandlerTest.java | 9 +++++++-- .../iguana/cc/query/source/FolderQuerySourceTest.java | 4 +++- .../test/resources/query/source/query-folder/query3.txt | 3 +++ 4 files changed, 16 insertions(+), 6 deletions(-) create mode 100644 iguana.corecontroller/src/test/resources/query/source/query-folder/query3.txt diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/handler/QueryHandler.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/handler/QueryHandler.java index 585e63fee..61b87f206 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/handler/QueryHandler.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/handler/QueryHandler.java @@ -82,6 +82,9 @@ private QuerySource getQuerySource() throws IOException { if (f.equals("separator")) { // No custom separator given -> use "###" return new FileSeparatorQuerySource(this.location); } + if (f.equals("folder")) { + return new FolderQuerySource(this.location); + } LOGGER.error("Unknown query format: {}", f); } @@ -92,9 +95,6 @@ private QuerySource getQuerySource() throws IOException { String separator = (String) format.get("separator"); return new FileSeparatorQuerySource(this.location, separator); } - if (format.containsKey("folder")) { - return new FolderQuerySource(this.location); - } LOGGER.error("Unknown query format: {}", format); } diff --git a/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/handler/QueryHandlerTest.java b/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/handler/QueryHandlerTest.java index 78008464d..2dae03817 100644 --- a/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/handler/QueryHandlerTest.java +++ b/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/handler/QueryHandlerTest.java @@ -31,8 +31,14 @@ public static Collection data() { Collection testData = new ArrayList<>(); // Defaults: one-per-line, caching, linear + Map config0 = new HashMap<>(); + config0.put("location", "src/test/resources/query/source/queries.txt"); + testData.add(new Object[]{config0, linear}); + + // Defaults: caching, linear Map config1 = new HashMap<>(); - config1.put("location", "src/test/resources/query/source/queries.txt"); + config1.put("location", "src/test/resources/query/source/query-folder"); + config1.put("format", "folder"); testData.add(new Object[]{config1, linear}); // Defaults: separator("###"), caching, linear @@ -41,7 +47,6 @@ public static Collection data() { config2.put("format", "separator"); testData.add(new Object[]{config2, linear}); - Map config3 = new HashMap<>(); config3.put("location", "src/test/resources/query/source/separated-queries-default.txt"); Map format3 = new HashMap<>(); diff --git a/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/source/FolderQuerySourceTest.java b/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/source/FolderQuerySourceTest.java index ee87a55c5..4f52ee976 100644 --- a/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/source/FolderQuerySourceTest.java +++ b/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/source/FolderQuerySourceTest.java @@ -20,13 +20,14 @@ public FolderQuerySourceTest() { @Test public void size() { - assertEquals(2, this.querySource.size()); + assertEquals(3, this.querySource.size()); } @Test public void getQuery() throws IOException { assertEquals("QUERY 1 {still query 1}", this.querySource.getQuery(0)); assertEquals("QUERY 2 {still query 2}", this.querySource.getQuery(1)); + assertEquals("QUERY 3 {still query 3}", this.querySource.getQuery(2)); } @Test @@ -34,6 +35,7 @@ public void getAllQueries() throws IOException { List expected = new ArrayList<>(3); expected.add("QUERY 1 {still query 1}"); expected.add("QUERY 2 {still query 2}"); + expected.add("QUERY 3 {still query 3}"); assertEquals(expected, this.querySource.getAllQueries()); } diff --git a/iguana.corecontroller/src/test/resources/query/source/query-folder/query3.txt b/iguana.corecontroller/src/test/resources/query/source/query-folder/query3.txt new file mode 100644 index 000000000..e34d54dad --- /dev/null +++ b/iguana.corecontroller/src/test/resources/query/source/query-folder/query3.txt @@ -0,0 +1,3 @@ +QUERY 3 { +still query 3 +} \ No newline at end of file From aecae1a488254750cdad89962622b231d66954ca Mon Sep 17 00:00:00 2001 From: frensing Date: Tue, 25 Oct 2022 22:27:48 +0300 Subject: [PATCH 11/47] add hashcode and triplestats generation --- .../iguana/cc/query/handler/QueryHandler.java | 58 +++++++++++++++++-- .../{ => impl}/LinearQuerySelector.java | 3 +- .../{ => impl}/RandomQuerySelector.java | 3 +- .../set/{newimpl => }/AbstractQuerySet.java | 8 ++- .../aksw/iguana/cc/query/set/QuerySet.java | 3 + .../cc/query/set/impl/FileBasedQuerySet.java | 5 ++ .../cc/query/set/impl/InMemQuerySet.java | 5 ++ .../query/set/newimpl/FileBasedQuerySet.java | 1 + .../cc/query/set/newimpl/InMemQuerySet.java | 1 + .../cc/query/source/AbstractQuerySource.java | 17 ++++++ .../iguana/cc/query/source/QuerySource.java | 2 + .../{ => impl}/FileLineQuerySource.java | 8 +-- .../{ => impl}/FileSeparatorQuerySource.java | 8 +-- .../source/{ => impl}/FolderQuerySource.java | 14 +++-- .../cc/query/handler/QueryHandlerTest.java | 2 +- .../{ => impl}/LinearQuerySelectorTest.java | 4 +- .../{ => impl}/FileLineQuerySourceTest.java | 15 +++-- .../FileSeparatorQuerySourceTest.java | 23 ++++++-- .../{ => impl}/FolderQuerySourceTest.java | 15 +++-- 19 files changed, 156 insertions(+), 39 deletions(-) rename iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/selector/{ => impl}/LinearQuerySelector.java (79%) rename iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/selector/{ => impl}/RandomQuerySelector.java (80%) rename iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/{newimpl => }/AbstractQuerySet.java (78%) create mode 100644 iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/AbstractQuerySource.java rename iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/{ => impl}/FileLineQuerySource.java (79%) rename iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/{ => impl}/FileSeparatorQuerySource.java (94%) rename iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/{ => impl}/FolderQuerySource.java (81%) rename iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/selector/{ => impl}/LinearQuerySelectorTest.java (83%) rename iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/source/{ => impl}/FileLineQuerySourceTest.java (72%) rename iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/source/{ => impl}/FileSeparatorQuerySourceTest.java (70%) rename iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/source/{ => impl}/FolderQuerySourceTest.java (70%) diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/handler/QueryHandler.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/handler/QueryHandler.java index 61b87f206..2ffda73e5 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/handler/QueryHandler.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/handler/QueryHandler.java @@ -1,19 +1,27 @@ package org.aksw.iguana.cc.query.handler; -import org.aksw.iguana.cc.query.selector.LinearQuerySelector; +import org.aksw.iguana.cc.lang.LanguageProcessor; +import org.aksw.iguana.cc.lang.QueryWrapper; import org.aksw.iguana.cc.query.selector.QuerySelector; -import org.aksw.iguana.cc.query.selector.RandomQuerySelector; +import org.aksw.iguana.cc.query.selector.impl.LinearQuerySelector; +import org.aksw.iguana.cc.query.selector.impl.RandomQuerySelector; import org.aksw.iguana.cc.query.set.QuerySet; import org.aksw.iguana.cc.query.set.newimpl.FileBasedQuerySet; import org.aksw.iguana.cc.query.set.newimpl.InMemQuerySet; -import org.aksw.iguana.cc.query.source.FileLineQuerySource; -import org.aksw.iguana.cc.query.source.FileSeparatorQuerySource; -import org.aksw.iguana.cc.query.source.FolderQuerySource; import org.aksw.iguana.cc.query.source.QuerySource; +import org.aksw.iguana.cc.query.source.impl.FileLineQuerySource; +import org.aksw.iguana.cc.query.source.impl.FileSeparatorQuerySource; +import org.aksw.iguana.cc.query.source.impl.FolderQuerySource; +import org.aksw.iguana.cc.utils.FileUtils; +import org.aksw.iguana.commons.factory.TypedFactory; +import org.apache.jena.rdf.model.Model; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; import java.util.Map; public class QueryHandler { @@ -23,6 +31,7 @@ public class QueryHandler { protected Map config; protected Integer workerID; protected String location; + protected int hashcode; protected boolean caching; @@ -30,14 +39,22 @@ public class QueryHandler { protected QuerySet querySet; + protected LanguageProcessor langProcessor; + + protected String outputFolder; + public QueryHandler(Map config, Integer workerID) { this.config = config; this.workerID = workerID; this.location = (String) config.get("location"); + this.hashcode = FileUtils.getHashcodeFromFileContent(this.location); + + this.outputFolder = (String) config.get("outputFolder"); initQuerySet(); initQuerySelector(); + initLanguageProcessor(); // TODO pattern } @@ -45,7 +62,23 @@ public QueryHandler(Map config, Integer workerID) { public void getNextQuery(StringBuilder queryStr, StringBuilder queryID) throws IOException { int queryIndex = this.querySelector.getNextIndex(); queryStr.append(this.querySet.getQueryAtPos(queryIndex)); - queryID.append(this.querySet.getName()).append(":").append(queryIndex); + queryID.append(getQueryId(queryIndex)); + } + + public Model getTripleStats(String taskID) { + List queries = new ArrayList<>(this.querySet.size()); + for (int i = 0; i < this.querySet.size(); i++) { + try { + queries.add(new QueryWrapper(this.querySet.getQueryAtPos(i), getQueryId(i))); + } catch (Exception e) { + LOGGER.error("Could not parse query " + this.querySet.getName() + ":" + i, e); + } + } + return this.langProcessor.generateTripleStats(queries, "" + this.hashcode, taskID); + } + + public int getHashcode() { + return this.hashcode; } private void initQuerySet() { @@ -131,4 +164,17 @@ private void initQuerySelector() { LOGGER.error("Unknown order: " + order); } } + + private void initLanguageProcessor() { + Object langObj = this.config.getOrDefault("lang", "lang.SPARQL"); + if (langObj instanceof String) { + this.langProcessor = new TypedFactory().create((String) langObj, new HashMap<>()); + } else { + LOGGER.error("Unknown language: " + langObj); + } + } + + private String getQueryId(int i) { + return this.querySet.getName() + ":" + i; + } } diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/selector/LinearQuerySelector.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/selector/impl/LinearQuerySelector.java similarity index 79% rename from iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/selector/LinearQuerySelector.java rename to iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/selector/impl/LinearQuerySelector.java index 52d089a37..989eddd26 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/selector/LinearQuerySelector.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/selector/impl/LinearQuerySelector.java @@ -1,5 +1,6 @@ -package org.aksw.iguana.cc.query.selector; +package org.aksw.iguana.cc.query.selector.impl; +import org.aksw.iguana.cc.query.selector.AbstractQuerySelector; import org.aksw.iguana.commons.annotation.Shorthand; @Shorthand("LinearQuerySelector") diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/selector/RandomQuerySelector.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/selector/impl/RandomQuerySelector.java similarity index 80% rename from iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/selector/RandomQuerySelector.java rename to iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/selector/impl/RandomQuerySelector.java index fda40effc..c835f894b 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/selector/RandomQuerySelector.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/selector/impl/RandomQuerySelector.java @@ -1,5 +1,6 @@ -package org.aksw.iguana.cc.query.selector; +package org.aksw.iguana.cc.query.selector.impl; +import org.aksw.iguana.cc.query.selector.AbstractQuerySelector; import org.aksw.iguana.commons.annotation.Shorthand; import java.util.Random; diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/newimpl/AbstractQuerySet.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/AbstractQuerySet.java similarity index 78% rename from iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/newimpl/AbstractQuerySet.java rename to iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/AbstractQuerySet.java index 7ae890fb9..a03768656 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/newimpl/AbstractQuerySet.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/AbstractQuerySet.java @@ -1,6 +1,5 @@ -package org.aksw.iguana.cc.query.set.newimpl; +package org.aksw.iguana.cc.query.set; -import org.aksw.iguana.cc.query.set.QuerySet; import org.aksw.iguana.cc.query.source.QuerySource; public abstract class AbstractQuerySet implements QuerySet { @@ -22,4 +21,9 @@ public int size() { public String getName() { return this.name; } + + @Override + public int getHashcode() { + return this.querySource.getHashcode(); + } } diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/QuerySet.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/QuerySet.java index cbae48a6d..124f8aa40 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/QuerySet.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/QuerySet.java @@ -10,6 +10,7 @@ public interface QuerySet { /** * Gets a query at the position pos. + * * @param pos Position of the query in the set * @return The query at position pos */ @@ -18,4 +19,6 @@ public interface QuerySet { int size(); String getName(); + + int getHashcode(); } diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/impl/FileBasedQuerySet.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/impl/FileBasedQuerySet.java index 1adace21e..470ef7382 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/impl/FileBasedQuerySet.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/impl/FileBasedQuerySet.java @@ -38,4 +38,9 @@ public int size() { public String getName() { return queryFile.getName(); } + + @Override + public int getHashcode() { + return queryFile.hashCode(); + } } diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/impl/InMemQuerySet.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/impl/InMemQuerySet.java index 8fd63b78a..3a71b63a5 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/impl/InMemQuerySet.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/impl/InMemQuerySet.java @@ -29,4 +29,9 @@ public int size() { public String getName() { return name; } + + @Override + public int getHashcode() { + return queries.hashCode(); + } } diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/newimpl/FileBasedQuerySet.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/newimpl/FileBasedQuerySet.java index 8bd276e59..b819e4fb6 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/newimpl/FileBasedQuerySet.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/newimpl/FileBasedQuerySet.java @@ -1,5 +1,6 @@ package org.aksw.iguana.cc.query.set.newimpl; +import org.aksw.iguana.cc.query.set.AbstractQuerySet; import org.aksw.iguana.cc.query.source.QuerySource; import java.io.IOException; diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/newimpl/InMemQuerySet.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/newimpl/InMemQuerySet.java index a29c3379b..f1dbc35f1 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/newimpl/InMemQuerySet.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/newimpl/InMemQuerySet.java @@ -1,5 +1,6 @@ package org.aksw.iguana.cc.query.set.newimpl; +import org.aksw.iguana.cc.query.set.AbstractQuerySet; import org.aksw.iguana.cc.query.source.QuerySource; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/AbstractQuerySource.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/AbstractQuerySource.java new file mode 100644 index 000000000..23d5bffb3 --- /dev/null +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/AbstractQuerySource.java @@ -0,0 +1,17 @@ +package org.aksw.iguana.cc.query.source; + +import org.aksw.iguana.cc.utils.FileUtils; + +public abstract class AbstractQuerySource implements QuerySource { + + protected String path; + + public AbstractQuerySource(String path) { + this.path = path; + } + + @Override + public int getHashcode() { + return FileUtils.getHashcodeFromFileContent(this.path); + } +} diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/QuerySource.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/QuerySource.java index 365315ccb..342bf7a55 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/QuerySource.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/QuerySource.java @@ -10,4 +10,6 @@ public interface QuerySource { String getQuery(int index) throws IOException; List getAllQueries() throws IOException; + + int getHashcode(); } diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/FileLineQuerySource.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/impl/FileLineQuerySource.java similarity index 79% rename from iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/FileLineQuerySource.java rename to iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/impl/FileLineQuerySource.java index fa72e2667..f1ba31322 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/FileLineQuerySource.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/impl/FileLineQuerySource.java @@ -1,5 +1,6 @@ -package org.aksw.iguana.cc.query.source; +package org.aksw.iguana.cc.query.source.impl; +import org.aksw.iguana.cc.query.source.AbstractQuerySource; import org.aksw.iguana.cc.utils.FileUtils; import java.io.File; @@ -7,15 +8,14 @@ import java.nio.file.Files; import java.util.List; -public class FileLineQuerySource implements QuerySource { +public class FileLineQuerySource extends AbstractQuerySource { - protected String path; protected File queryFile; protected int size; public FileLineQuerySource(String path) throws IOException { - this.path = path; + super(path); this.queryFile = new File(this.path); this.size = FileUtils.countLines(this.queryFile); } diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/FileSeparatorQuerySource.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/impl/FileSeparatorQuerySource.java similarity index 94% rename from iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/FileSeparatorQuerySource.java rename to iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/impl/FileSeparatorQuerySource.java index 66f1023e3..3eb5a2482 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/FileSeparatorQuerySource.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/impl/FileSeparatorQuerySource.java @@ -1,5 +1,6 @@ -package org.aksw.iguana.cc.query.source; +package org.aksw.iguana.cc.query.source.impl; +import org.aksw.iguana.cc.query.source.AbstractQuerySource; import org.aksw.iguana.cc.utils.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -14,12 +15,11 @@ import java.util.List; import java.util.stream.Stream; -public class FileSeparatorQuerySource implements QuerySource { +public class FileSeparatorQuerySource extends AbstractQuerySource { private static final Logger LOGGER = LoggerFactory.getLogger(FileSeparatorQuerySource.class); private static final String DEFAULT_SEPARATOR = "###"; - protected String path; protected File queryFile; protected String separator; protected int size; @@ -31,7 +31,7 @@ public FileSeparatorQuerySource(String path) { } public FileSeparatorQuerySource(String path, String separator) { - this.path = path; + super(path); this.queryFile = new File(this.path); this.separator = separator; diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/FolderQuerySource.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/impl/FolderQuerySource.java similarity index 81% rename from iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/FolderQuerySource.java rename to iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/impl/FolderQuerySource.java index 1c9e65ba2..1d95ef813 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/FolderQuerySource.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/impl/FolderQuerySource.java @@ -1,5 +1,7 @@ -package org.aksw.iguana.cc.query.source; +package org.aksw.iguana.cc.query.source.impl; +import org.aksw.iguana.cc.query.source.AbstractQuerySource; +import org.aksw.iguana.cc.utils.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -10,15 +12,14 @@ import java.util.ArrayList; import java.util.List; -public class FolderQuerySource implements QuerySource { +public class FolderQuerySource extends AbstractQuerySource { protected static final Logger LOGGER = LoggerFactory.getLogger(FolderQuerySource.class); - protected String path; protected File[] files; public FolderQuerySource(String path) { - this.path = path; + super(path); indexFolder(); } @@ -63,4 +64,9 @@ public List getAllQueries() throws IOException { } return queries; } + + @Override + public int getHashcode() { + return FileUtils.getHashcodeFromFileContent(this.files[0].getAbsolutePath()); + } } diff --git a/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/handler/QueryHandlerTest.java b/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/handler/QueryHandlerTest.java index 2dae03817..6eca1ca80 100644 --- a/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/handler/QueryHandlerTest.java +++ b/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/handler/QueryHandlerTest.java @@ -68,7 +68,7 @@ public static Collection data() { } @Test - public void getNextQuery() throws IOException { + public void getNextQueryTest() throws IOException { StringBuilder query = new StringBuilder(); StringBuilder queryID = new StringBuilder(); this.queryHandler.getNextQuery(query, queryID); diff --git a/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/selector/LinearQuerySelectorTest.java b/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/selector/impl/LinearQuerySelectorTest.java similarity index 83% rename from iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/selector/LinearQuerySelectorTest.java rename to iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/selector/impl/LinearQuerySelectorTest.java index 52e5e4cfd..8f6d18947 100644 --- a/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/selector/LinearQuerySelectorTest.java +++ b/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/selector/impl/LinearQuerySelectorTest.java @@ -1,4 +1,4 @@ -package org.aksw.iguana.cc.query.selector; +package org.aksw.iguana.cc.query.selector.impl; import org.junit.Before; import org.junit.Test; @@ -15,7 +15,7 @@ public void setUp() { } @Test - public void getNextIndex() { + public void getNextIndexTest() { for (int i = 0; i < 10; i++) { assertEquals(i % 5, this.linearQuerySelector.getNextIndex()); } diff --git a/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/source/FileLineQuerySourceTest.java b/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/source/impl/FileLineQuerySourceTest.java similarity index 72% rename from iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/source/FileLineQuerySourceTest.java rename to iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/source/impl/FileLineQuerySourceTest.java index be142841d..d9a969c7c 100644 --- a/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/source/FileLineQuerySourceTest.java +++ b/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/source/impl/FileLineQuerySourceTest.java @@ -1,5 +1,6 @@ -package org.aksw.iguana.cc.query.source; +package org.aksw.iguana.cc.query.source.impl; +import org.aksw.iguana.cc.utils.FileUtils; import org.junit.Test; import java.io.IOException; @@ -19,19 +20,19 @@ public FileLineQuerySourceTest() throws IOException { } @Test - public void size() { + public void sizeTest() { assertEquals(3, this.querySource.size()); } @Test - public void getQuery() throws IOException { + public void getQueryTest() throws IOException { assertEquals("QUERY 1 {still query 1}", this.querySource.getQuery(0)); assertEquals("QUERY 2 {still query 2}", this.querySource.getQuery(1)); assertEquals("QUERY 3 {still query 3}", this.querySource.getQuery(2)); } @Test - public void getAllQueries() throws IOException { + public void getAllQueriesTest() throws IOException { List expected = new ArrayList<>(3); expected.add("QUERY 1 {still query 1}"); expected.add("QUERY 2 {still query 2}"); @@ -39,4 +40,10 @@ public void getAllQueries() throws IOException { assertEquals(expected, this.querySource.getAllQueries()); } + + @Test + public void getHashcodeTest() { + int expected = FileUtils.getHashcodeFromFileContent(PATH); + assertEquals(expected, this.querySource.getHashcode()); + } } \ No newline at end of file diff --git a/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/source/FileSeparatorQuerySourceTest.java b/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/source/impl/FileSeparatorQuerySourceTest.java similarity index 70% rename from iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/source/FileSeparatorQuerySourceTest.java rename to iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/source/impl/FileSeparatorQuerySourceTest.java index 2a16ccaa7..b1c849330 100644 --- a/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/source/FileSeparatorQuerySourceTest.java +++ b/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/source/impl/FileSeparatorQuerySourceTest.java @@ -1,5 +1,6 @@ -package org.aksw.iguana.cc.query.source; +package org.aksw.iguana.cc.query.source.impl; +import org.aksw.iguana.cc.utils.FileUtils; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; @@ -16,11 +17,15 @@ public class FileSeparatorQuerySourceTest { private final FileSeparatorQuerySource querySource; + private final String path; + public FileSeparatorQuerySourceTest(String path, String separator) { + this.path = path; + if (separator == null) { - this.querySource = new FileSeparatorQuerySource(path); + this.querySource = new FileSeparatorQuerySource(this.path); } else { - this.querySource = new FileSeparatorQuerySource(path, separator); + this.querySource = new FileSeparatorQuerySource(this.path, separator); } } @@ -34,19 +39,19 @@ public static Collection data() { } @Test - public void size() { + public void sizeTest() { assertEquals(3, this.querySource.size()); } @Test - public void getQuery() throws IOException { + public void getQueryTest() throws IOException { assertEquals("QUERY 1 {still query 1}", this.querySource.getQuery(0)); assertEquals("QUERY 2 {still query 2}", this.querySource.getQuery(1)); assertEquals("QUERY 3 {still query 3}", this.querySource.getQuery(2)); } @Test - public void getAllQueries() throws IOException { + public void getAllQueriesTest() throws IOException { List expected = new ArrayList<>(3); expected.add("QUERY 1 {still query 1}"); expected.add("QUERY 2 {still query 2}"); @@ -54,4 +59,10 @@ public void getAllQueries() throws IOException { assertEquals(expected, this.querySource.getAllQueries()); } + + @Test + public void getHashcodeTest() { + int expected = FileUtils.getHashcodeFromFileContent(this.path); + assertEquals(expected, this.querySource.getHashcode()); + } } \ No newline at end of file diff --git a/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/source/FolderQuerySourceTest.java b/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/source/impl/FolderQuerySourceTest.java similarity index 70% rename from iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/source/FolderQuerySourceTest.java rename to iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/source/impl/FolderQuerySourceTest.java index 4f52ee976..6703e397f 100644 --- a/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/source/FolderQuerySourceTest.java +++ b/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/source/impl/FolderQuerySourceTest.java @@ -1,5 +1,6 @@ -package org.aksw.iguana.cc.query.source; +package org.aksw.iguana.cc.query.source.impl; +import org.aksw.iguana.cc.utils.FileUtils; import org.junit.Test; import java.io.IOException; @@ -19,19 +20,19 @@ public FolderQuerySourceTest() { } @Test - public void size() { + public void sizeTest() { assertEquals(3, this.querySource.size()); } @Test - public void getQuery() throws IOException { + public void getQueryTest() throws IOException { assertEquals("QUERY 1 {still query 1}", this.querySource.getQuery(0)); assertEquals("QUERY 2 {still query 2}", this.querySource.getQuery(1)); assertEquals("QUERY 3 {still query 3}", this.querySource.getQuery(2)); } @Test - public void getAllQueries() throws IOException { + public void getAllQueriesTest() throws IOException { List expected = new ArrayList<>(3); expected.add("QUERY 1 {still query 1}"); expected.add("QUERY 2 {still query 2}"); @@ -39,4 +40,10 @@ public void getAllQueries() throws IOException { assertEquals(expected, this.querySource.getAllQueries()); } + + @Test + public void getHashcodeTest() { + int expected = FileUtils.getHashcodeFromFileContent(PATH + "/query1.txt"); + assertEquals(expected, this.querySource.getHashcode()); + } } \ No newline at end of file From b0e23f21270db90d1ce55b8d916933f28a727885 Mon Sep 17 00:00:00 2001 From: frensing Date: Tue, 25 Oct 2022 23:18:55 +0300 Subject: [PATCH 12/47] gitignore --- .gitignore | 16 +++++++-------- .../aksw/iguana/cc/config/IguanaConfig.java | 20 +++++++++---------- 2 files changed, 17 insertions(+), 19 deletions(-) diff --git a/.gitignore b/.gitignore index dfd0dc545..b998c1d33 100644 --- a/.gitignore +++ b/.gitignore @@ -108,14 +108,14 @@ local.properties # When using Gradle or Maven with auto-import, you should exclude module files, # since they will be recreated, and may cause churn. Uncomment if using # auto-import. -# .idea/artifacts -# .idea/compiler.xml -# .idea/jarRepositories.xml -# .idea/modules.xml -# .idea/*.iml -# .idea/modules -# *.iml -# *.ipr +.idea/artifacts +.idea/compiler.xml +.idea/jarRepositories.xml +.idea/modules.xml +.idea/*.iml +.idea/modules +*.iml +*.ipr # CMake cmake-build-*/ diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/config/IguanaConfig.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/config/IguanaConfig.java index cdb672278..d6a4dfa68 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/config/IguanaConfig.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/config/IguanaConfig.java @@ -36,7 +36,6 @@ * Further on executes the pre and post script hooks, before and after a class. * Following values will be exchanged in the script string {{Connection}} {{Dataset.name}} {{Dataset.file}} {{taskID}} * - * * @author f.conrads * */ @@ -45,20 +44,19 @@ public class IguanaConfig { private static final Logger LOGGER = LoggerFactory .getLogger(IguanaConfig.class); - private String suiteID; @JsonProperty(required = true) private List datasets; @JsonProperty(required = true) private List connections; @JsonProperty(required = true) private List tasks; - @JsonProperty(required = false) + @JsonProperty() private String preScriptHook; - @JsonProperty(required = false) + @JsonProperty() private String postScriptHook; - @JsonProperty(required = false) + @JsonProperty() private List metrics; - @JsonProperty(required = false) + @JsonProperty() private List storages; @@ -73,7 +71,7 @@ public void start() throws ExecuteException, IOException { //get SuiteID String suiteID = generateSuiteID(); //generate ExpID - Integer expID = 0; + int expID = 0; for(Dataset dataset: datasets){ expID++; @@ -96,7 +94,7 @@ public void start() throws ExecuteException, IOException { ScriptExecutor.execSafe(execScript, args); } LOGGER.info("Executing Task [{}/{}: {}, {}, {}]", taskID, task.getName(), dataset.getName(), con.getName(), task.getClassName()); - controller.startTask(new String[]{suiteID, suiteID+"/"+expID.toString(), suiteID+"/"+expID.toString()+"/"+taskID.toString()}, dataset.getName(), SerializationUtils.clone(con), SerializationUtils.clone(task)); + controller.startTask(new String[]{suiteID, suiteID + "/" + expID, suiteID + "/" + expID + "/" + taskID}, dataset.getName(), SerializationUtils.clone(con), SerializationUtils.clone(task)); if(postScriptHook!=null){ String execScript = postScriptHook.replace("{{dataset.name}}", dataset.getName()) .replace("{{connection}}", con.getName()) @@ -133,7 +131,7 @@ private RPController initResultProcessor() { metrics.add(config); config = new MetricConfig(); config.setClassName(QPSMetric.class.getCanonicalName()); - Map configMap = new HashMap(); + Map configMap = new HashMap<>(); configMap.put("penalty", 180000); config.setConfiguration(configMap); metrics.add(config); @@ -149,12 +147,12 @@ private RPController initResultProcessor() { } //Create Storages - List storages = new ArrayList(); + List storages = new ArrayList<>(); for(StorageConfig config : this.storages){ storages.add(config.createStorage()); } //Create Metrics - List metrics = new ArrayList(); + List metrics = new ArrayList<>(); for(MetricConfig config : this.metrics){ metrics.add(config.createMetric()); } From f2a5699f745af0c393150b46c0b7905d9a2965d7 Mon Sep 17 00:00:00 2001 From: frensing Date: Tue, 25 Oct 2022 23:21:22 +0300 Subject: [PATCH 13/47] cleanup --- .../iguana/cc/controller/TaskController.java | 40 +++++++------------ 1 file changed, 14 insertions(+), 26 deletions(-) diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/controller/TaskController.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/controller/TaskController.java index 12838b94c..33483b144 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/controller/TaskController.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/controller/TaskController.java @@ -1,6 +1,3 @@ -/** - * - */ package org.aksw.iguana.cc.controller; import org.aksw.iguana.cc.config.elements.Connection; @@ -11,37 +8,28 @@ import org.slf4j.LoggerFactory; import java.io.IOException; -import java.util.HashMap; -import java.util.Map; import java.util.concurrent.TimeoutException; /** * Task Controlling, will start the actual benchmark tasks and its {@link org.aksw.iguana.cc.tasks.TaskManager} * - * - * * @author f.conrads - * */ public class TaskController { - private static Map shortHandMap = new HashMap(); - - private static final Logger LOGGER = LoggerFactory - .getLogger(TaskController.class); - - public void startTask(String[] ids, String dataset, Connection con, Task task) { - TaskManager tmanager = new TaskManager(); - String className=task.getClassName(); - TaskFactory factory = new TaskFactory(); - tmanager.setTask(factory.create(className, task.getConfiguration())); - try { - tmanager.startTask(ids, dataset, con, task.getName()); - } catch (IOException | TimeoutException e) { - LOGGER.error("Could not start Task "+className, e); - } - } - - + private static final Logger LOGGER = LoggerFactory + .getLogger(TaskController.class); + + public void startTask(String[] ids, String dataset, Connection con, Task task) { + TaskManager tmanager = new TaskManager(); + String className = task.getClassName(); + TaskFactory factory = new TaskFactory(); + tmanager.setTask(factory.create(className, task.getConfiguration())); + try { + tmanager.startTask(ids, dataset, con, task.getName()); + } catch (IOException | TimeoutException e) { + LOGGER.error("Could not start Task " + className, e); + } + } } From e6de3d650c107af9a841f544c938871435dcad1b Mon Sep 17 00:00:00 2001 From: frensing Date: Thu, 27 Oct 2022 12:39:59 +0300 Subject: [PATCH 14/47] fix order evaluation --- .../iguana/cc/query/handler/QueryHandler.java | 82 +++++++------------ .../cc/query/handler/QueryHandlerTest.java | 6 +- 2 files changed, 34 insertions(+), 54 deletions(-) diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/handler/QueryHandler.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/handler/QueryHandler.java index 2ffda73e5..35c56b2ba 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/handler/QueryHandler.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/handler/QueryHandler.java @@ -12,7 +12,6 @@ import org.aksw.iguana.cc.query.source.impl.FileLineQuerySource; import org.aksw.iguana.cc.query.source.impl.FileSeparatorQuerySource; import org.aksw.iguana.cc.query.source.impl.FolderQuerySource; -import org.aksw.iguana.cc.utils.FileUtils; import org.aksw.iguana.commons.factory.TypedFactory; import org.apache.jena.rdf.model.Model; import org.slf4j.Logger; @@ -28,7 +27,7 @@ public class QueryHandler { protected final Logger LOGGER = LoggerFactory.getLogger(QueryHandler.class); - protected Map config; + protected Map config; protected Integer workerID; protected String location; protected int hashcode; @@ -43,19 +42,19 @@ public class QueryHandler { protected String outputFolder; - public QueryHandler(Map config, Integer workerID) { + public QueryHandler(Map config, Integer workerID) { this.config = config; this.workerID = workerID; this.location = (String) config.get("location"); - this.hashcode = FileUtils.getHashcodeFromFileContent(this.location); - this.outputFolder = (String) config.get("outputFolder"); initQuerySet(); initQuerySelector(); initLanguageProcessor(); + this.hashcode = this.querySet.getHashcode(); + // TODO pattern } @@ -81,65 +80,43 @@ public int getHashcode() { return this.hashcode; } - private void initQuerySet() { - try { - QuerySource querySource = getQuerySource(); - - Boolean caching = (Boolean) config.get("caching"); - if (caching == null) { - caching = true; - } - this.caching = caching; - - if (this.caching) { - this.querySet = new InMemQuerySet(this.location, querySource); - } else { - this.querySet = new FileBasedQuerySet(this.location, querySource); - } - } catch (IOException e) { - LOGGER.error("Could not create QuerySource", e); - } + public int getQueryCount() { + return this.querySet.size(); } - private QuerySource getQuerySource() throws IOException { - Object formatObj = this.config.get("format"); - if (formatObj == null) { // Default - return new FileLineQuerySource(this.location); - } - - if (formatObj instanceof String) { - String f = (String) formatObj; - if (f.equals("one_per_line")) { - return new FileLineQuerySource(this.location); - } - if (f.equals("separator")) { // No custom separator given -> use "###" - return new FileSeparatorQuerySource(this.location); - } - if (f.equals("folder")) { - return new FolderQuerySource(this.location); - } + private void initQuerySet() { + this.caching = (Boolean) this.config.getOrDefault("caching", true); - LOGGER.error("Unknown query format: {}", f); + if (this.caching) { + this.querySet = new InMemQuerySet(this.location, createQuerySource()); + } else { + this.querySet = new FileBasedQuerySet(this.location, createQuerySource()); } + } + private QuerySource createQuerySource() { + Object formatObj = this.config.getOrDefault("format", "one-per-line"); if (formatObj instanceof Map) { Map format = (Map) formatObj; if (format.containsKey("separator")) { - String separator = (String) format.get("separator"); - return new FileSeparatorQuerySource(this.location, separator); + return new FileSeparatorQuerySource(this.location, (String) format.get("separator")); + } + } else { + switch ((String) formatObj) { + case "one-per-line": + return new FileLineQuerySource(this.location); + case "separator": + return new FileSeparatorQuerySource(this.location); + case "folder": + return new FolderQuerySource(this.location); } - - LOGGER.error("Unknown query format: {}", format); } - + LOGGER.error("Could not create QuerySource for format {}", formatObj); return null; } private void initQuerySelector() { - Object orderObj = this.config.get("order"); - if (orderObj == null) { // Default - this.querySelector = new LinearQuerySelector(this.querySet.size()); - } + Object orderObj = this.config.getOrDefault("order", "linear"); if (orderObj instanceof String) { String order = (String) orderObj; @@ -156,8 +133,9 @@ private void initQuerySelector() { } if (orderObj instanceof Map) { Map order = (Map) orderObj; - if (order.containsKey("seed")) { - Integer seed = (Integer) order.get("seed"); + if (order.containsKey("random")) { + Map random = (Map) order.get("random"); + Integer seed = (Integer) random.get("seed"); this.querySelector = new RandomQuerySelector(this.querySet.size(), seed); return; } diff --git a/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/handler/QueryHandlerTest.java b/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/handler/QueryHandlerTest.java index 6eca1ca80..bd54473a0 100644 --- a/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/handler/QueryHandlerTest.java +++ b/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/handler/QueryHandlerTest.java @@ -18,7 +18,7 @@ public class QueryHandlerTest { private final QueryHandler queryHandler; private final String[] expected; - public QueryHandlerTest(Map config, String[] expected) { + public QueryHandlerTest(Map config, String[] expected) { this.queryHandler = new QueryHandler(config, 0); // workerID 0 results in correct seed for RandomSelector this.expected = expected; } @@ -59,8 +59,10 @@ public static Collection data() { // Defaults: one-per-line, caching Map config4 = new HashMap<>(); config4.put("location", "src/test/resources/query/source/queries.txt"); + Map random4 = new HashMap<>(); + random4.put("seed", 0); Map order4 = new HashMap<>(); - order4.put("seed", 0); + order4.put("random", random4); config4.put("order", order4); testData.add(new Object[]{config4, random}); From 75fece401fb5c35c35f31cc1cf49874dd77161ba Mon Sep 17 00:00:00 2001 From: frensing Date: Mon, 31 Oct 2022 18:27:45 +0200 Subject: [PATCH 15/47] refactor httpworkers to use new query handling --- .../iguana/cc/query/handler/QueryHandler.java | 4 + .../impl/DelimInstancesQueryHandler.java | 86 --- .../source/impl/FileLineQuerySource.java | 11 +- .../source/impl/FileSeparatorQuerySource.java | 4 +- .../AbstractRandomQueryChooserWorker.java | 45 -- .../aksw/iguana/cc/worker/AbstractWorker.java | 561 +++++++++--------- .../org/aksw/iguana/cc/worker/Worker.java | 35 +- .../iguana/cc/worker/impl/HttpGetWorker.java | 30 +- .../iguana/cc/worker/impl/HttpPostWorker.java | 7 +- .../iguana/cc/worker/impl/HttpWorker.java | 133 +++-- .../iguana/cc/worker/impl/SPARQLWorker.java | 5 +- .../iguana/cc/worker/impl/UPDATEWorker.java | 218 ++++--- .../impl/DelimInstancesQueryHandlerTest.java | 120 ---- .../query/impl/InstancesQueryHandlerTest.java | 118 ---- .../iguana/cc/query/impl/UpdatePathTest.java | 46 -- .../aksw/iguana/cc/worker/HTTPWorkerTest.java | 177 +++--- .../iguana/cc/worker/UPDATEWorkerTest.java | 114 ++-- .../cc/worker/impl/HttpPostWorkerTest.java | 10 +- 18 files changed, 657 insertions(+), 1067 deletions(-) delete mode 100644 iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/impl/DelimInstancesQueryHandler.java delete mode 100644 iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/AbstractRandomQueryChooserWorker.java delete mode 100644 iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/impl/DelimInstancesQueryHandlerTest.java delete mode 100644 iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/impl/InstancesQueryHandlerTest.java delete mode 100644 iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/impl/UpdatePathTest.java diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/handler/QueryHandler.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/handler/QueryHandler.java index 35c56b2ba..598e89b90 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/handler/QueryHandler.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/handler/QueryHandler.java @@ -84,6 +84,10 @@ public int getQueryCount() { return this.querySet.size(); } + public LanguageProcessor getLanguageProcessor() { + return this.langProcessor; + } + private void initQuerySet() { this.caching = (Boolean) this.config.getOrDefault("caching", true); diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/impl/DelimInstancesQueryHandler.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/impl/DelimInstancesQueryHandler.java deleted file mode 100644 index 70f956924..000000000 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/impl/DelimInstancesQueryHandler.java +++ /dev/null @@ -1,86 +0,0 @@ -package org.aksw.iguana.cc.query.impl; - -import org.aksw.iguana.cc.query.set.QuerySet; -import org.aksw.iguana.cc.query.set.impl.InMemQuerySet; -import org.aksw.iguana.cc.worker.Worker; -import org.aksw.iguana.commons.annotation.Shorthand; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.*; -import java.util.LinkedList; -import java.util.List; - -/** - * Uses a delimiter line to read one query - * default uses empty line - */ -@Shorthand("DelimInstancesQueryHandler") -public class DelimInstancesQueryHandler extends InstancesQueryHandler { - - private static final Logger LOGGER = LoggerFactory.getLogger(DelimInstancesQueryHandler.class); - - - private String delim= ""; - - public DelimInstancesQueryHandler(List workers) { - super(workers); - } - - - public DelimInstancesQueryHandler(String delim, List workers) { - super(workers); - this.delim = delim; - } - - - public DelimInstancesQueryHandler(List workers, String lang) { - super(workers, lang); - } - - public DelimInstancesQueryHandler(List workers, String lang, String delim) { - super(workers, lang); - this.delim = delim; - } - - @Override - protected QuerySet[] generateUpdatesPerLine(String updatePath, String idPrefix, int hashcode) { - return generateQueryPerLine(updatePath, idPrefix, hashcode); - } - - @Override - protected QuerySet[] generateQueryPerLine(String queryFileName, String idPrefix, int hashcode) { - - File queryFile = new File(queryFileName); - List ret = new LinkedList(); - try ( - BufferedReader reader = new BufferedReader(new FileReader(queryFileName))) { - StringBuilder currentQuery = new StringBuilder(); - String queryStr; - int id = 0; - while ((queryStr = reader.readLine()) != null) { - if (queryStr.equals(delim)) { - if(currentQuery.toString().trim().isEmpty()){ - currentQuery = new StringBuilder(); - continue; - } - ret.add(new InMemQuerySet(idPrefix + id++, getInstances(currentQuery.toString().trim()))); - currentQuery = new StringBuilder(); - continue; - } - currentQuery.append(queryStr).append("\n"); - - } - if(!currentQuery.toString().trim().isEmpty()) { - ret.add(new InMemQuerySet(idPrefix + id++, getInstances(currentQuery.toString()))); - } - currentQuery = new StringBuilder(); - } catch (IOException e) { - LOGGER.error("could not read queries"); - } - return ret.toArray(new QuerySet[]{}); - } - - - -} diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/impl/FileLineQuerySource.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/impl/FileLineQuerySource.java index f1ba31322..37e92e062 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/impl/FileLineQuerySource.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/impl/FileLineQuerySource.java @@ -2,6 +2,8 @@ import org.aksw.iguana.cc.query.source.AbstractQuerySource; import org.aksw.iguana.cc.utils.FileUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.File; import java.io.IOException; @@ -9,15 +11,20 @@ import java.util.List; public class FileLineQuerySource extends AbstractQuerySource { + private static final Logger LOGGER = LoggerFactory.getLogger(FileLineQuerySource.class); protected File queryFile; protected int size; - public FileLineQuerySource(String path) throws IOException { + public FileLineQuerySource(String path) { super(path); this.queryFile = new File(this.path); - this.size = FileUtils.countLines(this.queryFile); + try { + this.size = FileUtils.countLines(this.queryFile); + } catch (IOException e) { + LOGGER.error("Could not read queries"); + } } @Override diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/impl/FileSeparatorQuerySource.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/impl/FileSeparatorQuerySource.java index 3eb5a2482..572d5bdd9 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/impl/FileSeparatorQuerySource.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/impl/FileSeparatorQuerySource.java @@ -41,8 +41,7 @@ public FileSeparatorQuerySource(String path, String separator) { private void indexFile() { this.separatorPositions = new LinkedList<>(); int separatorCount = 0; - try { - BufferedReader reader = FileUtils.getBufferedReader(this.queryFile); + try (BufferedReader reader = FileUtils.getBufferedReader(this.queryFile)) { int index = 0; String line; this.separatorPositions.add(-1); @@ -54,7 +53,6 @@ private void indexFile() { index++; } this.separatorPositions.add(index); - reader.close(); } catch (IOException e) { LOGGER.error("Could not read queries"); } diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/AbstractRandomQueryChooserWorker.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/AbstractRandomQueryChooserWorker.java deleted file mode 100644 index 5c0f0f828..000000000 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/AbstractRandomQueryChooserWorker.java +++ /dev/null @@ -1,45 +0,0 @@ -package org.aksw.iguana.cc.worker; - -import org.aksw.iguana.cc.config.elements.Connection; -import org.aksw.iguana.cc.query.set.QuerySet; - -import java.io.IOException; -import java.util.Random; - -public abstract class AbstractRandomQueryChooserWorker extends AbstractWorker { - - protected int currentQueryID; - protected Random queryChooser; - - - public AbstractRandomQueryChooserWorker(String taskID, Connection connection, String queriesFile, Integer timeOut, Integer timeLimit, Integer fixedLatency, Integer gaussianLatency, Integer workerID) { - super(taskID, connection, queriesFile, timeOut, timeLimit, fixedLatency, gaussianLatency, workerID); - queryChooser = new Random(this.workerID); - - } - - @Override - public void setQueriesList(QuerySet[] queries) { - super.setQueriesList(queries); - this.currentQueryID = queryChooser.nextInt(this.queryFileList.length); - } - - - @Override - public void getNextQuery(StringBuilder queryStr, StringBuilder queryID) throws IOException { - // get next Query File and next random Query out of it. - QuerySet currentQueryFile = this.queryFileList[this.currentQueryID++]; - queryID.append(currentQueryFile.getName()); - - int queriesInFile = currentQueryFile.size(); - int queryLine = queryChooser.nextInt(queriesInFile); - queryStr.append(currentQueryFile.getQueryAtPos(queryLine)); - - // If there is no more query(Pattern) start from beginning. - if (this.currentQueryID >= this.queryFileList.length) { - this.currentQueryID = 0; - } - - } - -} diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/AbstractWorker.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/AbstractWorker.java index a637855c8..004f66797 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/AbstractWorker.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/AbstractWorker.java @@ -3,8 +3,8 @@ import org.aksw.iguana.cc.config.CONSTANTS; import org.aksw.iguana.cc.config.elements.Connection; import org.aksw.iguana.cc.model.QueryExecutionStats; +import org.aksw.iguana.cc.query.handler.QueryHandler; import org.aksw.iguana.cc.query.set.QuerySet; -import org.aksw.iguana.cc.utils.FileUtils; import org.aksw.iguana.commons.annotation.Nullable; import org.aksw.iguana.commons.annotation.Shorthand; import org.aksw.iguana.commons.constants.COMMON; @@ -25,10 +25,7 @@ import java.net.URI; import java.net.URISyntaxException; import java.time.Instant; -import java.util.Collection; -import java.util.LinkedList; -import java.util.Properties; -import java.util.Random; +import java.util.*; /** @@ -39,282 +36,284 @@ * and how to test this query. * * @author f.conrads - * */ public abstract class AbstractWorker implements Worker { - - /** - * Logger which should be used - */ - protected static final Logger LOGGER = LoggerFactory.getLogger(AbstractWorker.class); - - protected boolean endSignal = false; - protected long executedQueries; - - private Collection results = new LinkedList<>(); - protected String taskID; - - /** - * The worker Type. f.e. SPARQL or UPDATE or SQL or whatever - * Determined by the Shorthand of the class, if no Shorthand is provided the class name is used. - * The workerType is only used in logging messages. - */ - protected String workerType; - /** - * The unique ID of the worker, should be from 0 to n - */ - protected Integer workerID; - protected Properties extra = new Properties(); - - private Integer fixedLatency=0; - - private Integer gaussianLatency=0; - - private Random latencyRandomizer; - private Long endAtNOQM = null; - - /** - * List which contains all Files representing one query(Pattern) - */ - protected QuerySet[] queryFileList; - - protected Double timeLimit; - - protected Instant startTime; - - protected String queriesFileName; - - protected Connection con; - - protected Double timeOut=180000D; - - protected int queryHash; - - public AbstractWorker(String taskID, Connection connection, String queriesFile, @Nullable Integer timeOut, @Nullable Integer timeLimit, @Nullable Integer fixedLatency, @Nullable Integer gaussianLatency, Integer workerID) { - this.taskID = taskID; - this.workerID = workerID; - - if (this.getClass().getAnnotation(Shorthand.class) != null) { - this.workerType = this.getClass().getAnnotation(Shorthand.class).value(); - } else { - this.workerType = this.getClass().getName(); - } - - this.con = connection; - if (timeLimit != null) { - this.timeLimit = timeLimit.doubleValue(); - } - latencyRandomizer = new Random(this.workerID); - if (timeOut != null) - this.timeOut = timeOut.doubleValue(); - // Add latency Specs, add defaults - if (fixedLatency != null) - this.fixedLatency = fixedLatency; - if(gaussianLatency!=null) - this.gaussianLatency = gaussianLatency; - // set Query file/folder Name - this.queriesFileName = queriesFile; - LOGGER.debug("Initialized new Worker[{{}} : {{}}] for taskID {{}}", workerType, workerID, taskID); - } - - - @Override - public void waitTimeMs() { - double wait = this.fixedLatency.doubleValue(); - double gaussian = latencyRandomizer.nextDouble(); - wait += (gaussian * 2) * this.gaussianLatency; - LOGGER.debug("Worker[{} : {}]: Time to wait for next Query {}", workerType, workerID, wait); - try { - if(wait>0) - Thread.sleep((int) wait); - } catch (InterruptedException e) { - LOGGER.error("Worker[{{}} : {}]: Could not wait time before next query due to: {}", workerType, - workerID, e); - } - } - - /** - * This will start the worker. It will get the next query, wait as long as it - * should wait before executing the next query, then it will test the query and - * send it if not aborted yet to the ResultProcessor Module - * - */ - public void startWorker() { - // set extra meta key to send late - this.extra = new Properties(); - this.extra.put(CONSTANTS.WORKER_ID_KEY, workerID); - this.extra.setProperty(CONSTANTS.WORKER_TYPE_KEY, workerType); - this.extra.put(CONSTANTS.WORKER_TIMEOUT_MS, timeOut); - if(this.queryFileList!=null) - this.extra.put(COMMON.NO_OF_QUERIES, this.queryFileList.length); - // For Update and Logging purpose get startTime of Worker - this.startTime = Instant.now(); - - this.queryHash = FileUtils.getHashcodeFromFileContent(this.queriesFileName); - - LOGGER.info("Starting Worker[{{}} : {{}}].", this.workerType, this.workerID); - // Execute Queries as long as the Stresstest will need. - while (!this.endSignal && !hasExecutedNoOfQueryMixes(this.endAtNOQM)) { - // Get next query - StringBuilder query = new StringBuilder(); - StringBuilder queryID = new StringBuilder(); - try { - getNextQuery(query, queryID); - // check if endsignal was triggered - if (this.endSignal) { - break; - } - } catch (IOException e) { - LOGGER.error( - "Worker[{{}} : {{}}] : Something went terrible wrong in getting the next query. Worker will be shut down.", - this.workerType, this.workerID); - LOGGER.error("Error which occured:_", e); - break; - } - // Simulate Network Delay (or whatever should be simulated) - waitTimeMs(); - - // benchmark query - try { - executeQuery(query.toString(), queryID.toString()); - } catch (Exception e) { - LOGGER.error("Worker[{{}} : {{}}] : ERROR with query: {{}}", this.workerType, this.workerID, query); - } - //this.executedQueries++; - } - LOGGER.info("Stopping Worker[{{}} : {{}}].", this.workerType, this.workerID); - } - - protected HttpContext getAuthContext(String endpoint){ - HttpClientContext context = HttpClientContext.create(); - - if(con.getPassword()!=null && con.getUser()!=null && !con.getPassword().isEmpty() && !con.getUser().isEmpty()) { - CredentialsProvider provider = new BasicCredentialsProvider(); - - provider.setCredentials(new AuthScope(AuthScope.ANY_HOST, AuthScope.ANY_PORT), - new UsernamePasswordCredentials(con.getUser(), con.getPassword())); - - //create target host - String targetHost = endpoint; - try { - URI uri = new URI(endpoint); - targetHost = uri.getScheme() + "://" + uri.getHost() + ":" + uri.getPort(); - } catch (URISyntaxException e) { - e.printStackTrace(); - } - //set Auth cache - AuthCache authCache = new BasicAuthCache(); - BasicScheme basicAuth = new BasicScheme(); - authCache.put(HttpHost.create(targetHost), basicAuth); - - context.setCredentialsProvider(provider); - context.setAuthCache(authCache); - - } - return context; - } - - public synchronized void addResults(QueryExecutionStats results) - { - if (!this.endSignal && !hasExecutedNoOfQueryMixes(this.endAtNOQM)) { - // create Properties store it in List - Properties result = new Properties(); - result.setProperty(COMMON.EXPERIMENT_TASK_ID_KEY, this.taskID); - result.put(COMMON.RECEIVE_DATA_TIME, results.getExecutionTime()); - result.put(COMMON.RECEIVE_DATA_SUCCESS, results.getResponseCode()); - result.put(COMMON.RECEIVE_DATA_SIZE, results.getResultSize()); - result.put(COMMON.QUERY_HASH, queryHash); - result.setProperty(COMMON.QUERY_ID_KEY, results.getQueryID()); - result.put(COMMON.PENALTY, this.timeOut); - // Add extra Meta Key, worker ID and worker Type - result.put(COMMON.EXTRA_META_KEY, this.extra); - setResults(result); - executedQueries++; - - // - if(getNoOfQueries() > 0 && getExecutedQueries() % getNoOfQueries() == 0 ){ - LOGGER.info("Worker executed {} queryMixes", getExecutedQueries()*1.0/getNoOfQueries()); - } - } - } - - protected synchronized void setResults(Properties result) { - results.add(result); - } - - @Override - public synchronized Collection popQueryResults() { - if(results.isEmpty()){ - return null; - } - Collection ret = this.results; - this.results = new LinkedList<>(); - return ret; - } - - @Override - public long getExecutedQueries() { - return this.executedQueries; - } - - @Override - public void stopSending() { - this.endSignal = true; - LOGGER.debug("Worker[{{}} : {{}}] got stop signal.", workerType, workerID); - } - - @Override - public boolean isTerminated(){ - return this.endSignal; - } - - - @Override - public void run() { - startWorker(); - } - - /** - * Returns the name of the queries file name/update path - * - * @return file name/update path - */ - public String getQueriesFileName() { - return this.queriesFileName; - } - - /** - * Sets the Query Instances repr. in Files. - * - * @param queries - * File containing the query instances. - */ - public void setQueriesList(QuerySet[] queries) { - this.queryFileList = queries; - } - - /** - * The number of Queries in one mix - * - * @return - */ - public long getNoOfQueries() { - if(this.queryFileList == null){ - return 0; - } - return this.queryFileList.length; - } - - @Override - public boolean hasExecutedNoOfQueryMixes(Long noOfQueryMixes){ - if(noOfQueryMixes==null){ - return false; - } - return getExecutedQueries() / (getNoOfQueries() * 1.0) >= noOfQueryMixes; - } - - @Override - public void endAtNoOfQueryMixes(Long noOfQueryMixes){ - this.endAtNOQM=noOfQueryMixes; - } + protected static final Logger LOGGER = LoggerFactory.getLogger(AbstractWorker.class); + + protected String taskID; + + /** + * The unique ID of the worker, should be from 0 to n + */ + protected Integer workerID; + + /** + * The worker Type. f.e. SPARQL or UPDATE or SQL or whatever + * Determined by the Shorthand of the class, if no Shorthand is provided the class name is used. + * The workerType is only used in logging messages. + */ + protected String workerType; + protected Connection connection; + protected Map queries; + + protected Double timeLimit; + protected Double timeOut = 180000D; + protected Integer fixedLatency = 0; + protected Integer gaussianLatency = 0; + + protected boolean endSignal = false; + protected long executedQueries; + protected Properties extra = new Properties(); + protected Instant startTime; + protected Connection con; + protected int queryHash; + protected QueryHandler queryHandler; + private Collection results = new LinkedList<>(); + private Random latencyRandomizer; + private Long endAtNOQM = null; + + public AbstractWorker(String taskID, Integer workerID, Connection connection, Map queries, @Nullable Integer timeLimit, @Nullable Integer timeOut, @Nullable Integer fixedLatency, @Nullable Integer gaussianLatency) { + this.taskID = taskID; + this.workerID = workerID; + this.con = connection; + + handleTimeParams(timeLimit, timeOut, fixedLatency, gaussianLatency); + setWorkerType(); + + this.queryHandler = new QueryHandler(queries, this.workerID); + + LOGGER.debug("Initialized new Worker[{{}} : {{}}] for taskID {{}}", this.workerType, workerID, taskID); + } + + public AbstractWorker(String taskID, Connection connection, String queriesFile, @Nullable Integer timeOut, @Nullable Integer timeLimit, @Nullable Integer fixedLatency, @Nullable Integer gaussianLatency, Integer workerID) { + // FIXME use other constructor in child classes + } + + + @Override + public void waitTimeMs() { + double wait = this.fixedLatency.doubleValue(); + double gaussian = this.latencyRandomizer.nextDouble(); + wait += (gaussian * 2) * this.gaussianLatency; + LOGGER.debug("Worker[{} : {}]: Time to wait for next Query {}", this.workerType, this.workerID, wait); + try { + if (wait > 0) + Thread.sleep((int) wait); + } catch (InterruptedException e) { + LOGGER.error("Worker[{{}} : {}]: Could not wait time before next query due to: {}", this.workerType, this.workerID, e); + } + } + + /** + * This will start the worker. It will get the next query, wait as long as it + * should wait before executing the next query, then it will test the query and + * send it if not aborted yet to the ResultProcessor Module + */ + public void startWorker() { + // set extra meta key to send late + this.extra = new Properties(); + this.extra.put(CONSTANTS.WORKER_ID_KEY, this.workerID); + this.extra.setProperty(CONSTANTS.WORKER_TYPE_KEY, this.workerType); + this.extra.put(CONSTANTS.WORKER_TIMEOUT_MS, this.timeOut); + this.extra.put(COMMON.NO_OF_QUERIES, this.queryHandler.getQueryCount()); + // For Update and Logging purpose get startTime of Worker + this.startTime = Instant.now(); + + this.queryHash = this.queryHandler.getHashcode(); + + LOGGER.info("Starting Worker[{{}} : {{}}].", this.workerType, this.workerID); + // Execute Queries as long as the Stresstest will need. + while (!this.endSignal && !hasExecutedNoOfQueryMixes(this.endAtNOQM)) { + // Get next query + StringBuilder query = new StringBuilder(); + StringBuilder queryID = new StringBuilder(); + try { + getNextQuery(query, queryID); + // check if endsignal was triggered + if (this.endSignal) { + break; + } + } catch (IOException e) { + LOGGER.error( + "Worker[{{}} : {{}}] : Something went terrible wrong in getting the next query. Worker will be shut down.", + this.workerType, this.workerID); + LOGGER.error("Error which occured:_", e); + break; + } + // Simulate Network Delay (or whatever should be simulated) + waitTimeMs(); + + // benchmark query + try { + executeQuery(query.toString(), queryID.toString()); + } catch (Exception e) { + LOGGER.error("Worker[{{}} : {{}}] : ERROR with query: {{}}", this.workerType, this.workerID, query); + } + //this.executedQueries++; + } + LOGGER.info("Stopping Worker[{{}} : {{}}].", this.workerType, this.workerID); + } + + @Override + public void getNextQuery(StringBuilder query, StringBuilder queryID) throws IOException { + this.queryHandler.getNextQuery(query, queryID); + } + + protected HttpContext getAuthContext(String endpoint) { + HttpClientContext context = HttpClientContext.create(); + + if (this.con.getPassword() != null && this.con.getUser() != null && !this.con.getPassword().isEmpty() && !this.con.getUser().isEmpty()) { + CredentialsProvider provider = new BasicCredentialsProvider(); + + provider.setCredentials(new AuthScope(AuthScope.ANY_HOST, AuthScope.ANY_PORT), + new UsernamePasswordCredentials(this.con.getUser(), this.con.getPassword())); + + //create target host + String targetHost = endpoint; + try { + URI uri = new URI(endpoint); + targetHost = uri.getScheme() + "://" + uri.getHost() + ":" + uri.getPort(); + } catch (URISyntaxException e) { + e.printStackTrace(); + } + //set Auth cache + AuthCache authCache = new BasicAuthCache(); + BasicScheme basicAuth = new BasicScheme(); + authCache.put(HttpHost.create(targetHost), basicAuth); + + context.setCredentialsProvider(provider); + context.setAuthCache(authCache); + + } + return context; + } + + public synchronized void addResults(QueryExecutionStats results) { + if (!this.endSignal && !hasExecutedNoOfQueryMixes(this.endAtNOQM)) { + // create Properties store it in List + Properties result = new Properties(); + result.setProperty(COMMON.EXPERIMENT_TASK_ID_KEY, this.taskID); + result.put(COMMON.RECEIVE_DATA_TIME, results.getExecutionTime()); + result.put(COMMON.RECEIVE_DATA_SUCCESS, results.getResponseCode()); + result.put(COMMON.RECEIVE_DATA_SIZE, results.getResultSize()); + result.put(COMMON.QUERY_HASH, queryHash); + result.setProperty(COMMON.QUERY_ID_KEY, results.getQueryID()); + result.put(COMMON.PENALTY, this.timeOut); + // Add extra Meta Key, worker ID and worker Type + result.put(COMMON.EXTRA_META_KEY, this.extra); + setResults(result); + this.executedQueries++; + + // + if (getNoOfQueries() > 0 && getExecutedQueries() % getNoOfQueries() == 0) { + LOGGER.info("Worker executed {} queryMixes", getExecutedQueries() * 1.0 / getNoOfQueries()); + } + } + } + + protected synchronized void setResults(Properties result) { + this.results.add(result); + } + + @Override + public synchronized Collection popQueryResults() { + if (this.results.isEmpty()) { + return null; + } + Collection ret = this.results; + this.results = new LinkedList<>(); + return ret; + } + + @Override + public long getExecutedQueries() { + return this.executedQueries; + } + + @Override + public void stopSending() { + this.endSignal = true; + LOGGER.debug("Worker[{{}} : {{}}] got stop signal.", this.workerType, this.workerID); + } + + @Override + public boolean isTerminated() { + return this.endSignal; + } + + + @Override + public void run() { + startWorker(); + } + + /** + * Returns the name of the queries file name/update path + * + * @return file name/update path + */ + public String getQueriesFileName() { + // FIXME remove + return ""; + } + + /** + * Sets the Query Instances repr. in Files. + * + * @param queries File containing the query instances. + */ + public void setQueriesList(QuerySet[] queries) { + // FIXME remove + } + + /** + * The number of Queries in one mix + * + * @return + */ + public long getNoOfQueries() { + return this.queryHandler.getQueryCount(); + } + + @Override + public boolean hasExecutedNoOfQueryMixes(Long noOfQueryMixes) { + if (noOfQueryMixes == null) { + return false; + } + return getExecutedQueries() / (getNoOfQueries() * 1.0) >= noOfQueryMixes; + } + + @Override + public void endAtNoOfQueryMixes(Long noOfQueryMixes) { + this.endAtNOQM = noOfQueryMixes; + } + + @Override + public QueryHandler getQueryHandler() { + return this.queryHandler; + } + + private void handleTimeParams(Integer timeLimit, Integer timeOut, Integer fixedLatency, Integer gaussianLatency) { + if (timeLimit != null) { + this.timeLimit = timeLimit.doubleValue(); + } + if (timeOut != null) { + this.timeOut = timeOut.doubleValue(); + } + if (fixedLatency != null) { + this.fixedLatency = fixedLatency; + } + if (gaussianLatency != null) { + this.gaussianLatency = gaussianLatency; + } + this.latencyRandomizer = new Random(this.workerID); + } + + private void setWorkerType() { + if (this.getClass().getAnnotation(Shorthand.class) != null) { + this.workerType = this.getClass().getAnnotation(Shorthand.class).value(); + } else { + this.workerType = this.getClass().getName(); + } + } } diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/Worker.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/Worker.java index 077f58d76..36ae5cd76 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/Worker.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/Worker.java @@ -1,5 +1,6 @@ package org.aksw.iguana.cc.worker; +import org.aksw.iguana.cc.query.handler.QueryHandler; import org.aksw.iguana.cc.tasks.impl.Stresstest; import java.io.IOException; @@ -14,54 +15,58 @@ */ public interface Worker extends Runnable{ - + /** * This method executes a query and adds the results to the Result Processor for proper result and metric calculations. * Note: Some of the Worker implementations employ background threads to process the result of the query. * Due to this, this method does not return anything and each implementation of this method must also add the * results to Result Processor within this method. This can be done by calling AbstractWorker.addResults(QueryExecutionStats) - * @param query The query which should be executed + * + * @param query The query which should be executed * @param queryID the ID of the query which should be executed */ - public void executeQuery(String query, String queryID); - + void executeQuery(String query, String queryID); + /** * This method saves the next query in the queryStr StringBuilder and * the query id in the queryID. - * + * * @param queryStr The query should be stored in here! - * @param queryID The queryID should be stored in here! - * @throws IOException + * @param queryID The queryID should be stored in here! + * @throws IOException */ - public void getNextQuery(StringBuilder queryStr, StringBuilder queryID) throws IOException; + void getNextQuery(StringBuilder queryStr, StringBuilder queryID) throws IOException; + + + QueryHandler getQueryHandler(); /** * This should stop the next sending process. * If an execution started before this method was called, but answered after, it should not be counted! */ - public void stopSending(); + void stopSending(); /** * This will simulate the Time in ms to wait before testing the next query. * It can be used to simulate network delay. */ - public void waitTimeMs(); + void waitTimeMs(); + - /** * This will return the amount of executed queries so far * * @return no. of executed queries */ - public long getExecutedQueries(); - + long getExecutedQueries(); + /** * Get and remove all internal stored results of finished queries - * + * * @return list of Properties to send to RabbitMQ */ - public Collection popQueryResults(); + Collection popQueryResults(); boolean isTerminated(); diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/impl/HttpGetWorker.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/impl/HttpGetWorker.java index 7d01dd8f1..0f81ed6f2 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/impl/HttpGetWorker.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/impl/HttpGetWorker.java @@ -1,18 +1,15 @@ package org.aksw.iguana.cc.worker.impl; import org.aksw.iguana.cc.config.elements.Connection; -import org.aksw.iguana.cc.lang.LanguageProcessor; import org.aksw.iguana.commons.annotation.Nullable; import org.aksw.iguana.commons.annotation.Shorthand; -import org.aksw.iguana.commons.factory.TypedFactory; import org.apache.http.HttpHeaders; import org.apache.http.client.config.RequestConfig; import org.apache.http.client.methods.HttpGet; -import java.io.UnsupportedEncodingException; import java.net.URLEncoder; import java.nio.charset.StandardCharsets; -import java.util.HashMap; +import java.util.Map; /** @@ -27,37 +24,34 @@ public class HttpGetWorker extends HttpWorker { protected String responseType = null; + public HttpGetWorker(String taskID, Integer workerID, Connection connection, Map queries, @Nullable Integer timeLimit, @Nullable Integer timeOut, @Nullable Integer fixedLatency, @Nullable Integer gaussianLatency, @Nullable String parameterName, @Nullable String responseType) { + super(taskID, workerID, connection, queries, timeLimit, timeOut, fixedLatency, gaussianLatency); - public HttpGetWorker(String taskID, Connection connection, String queriesFile, @Nullable String responseType, @Nullable String parameterName, @Nullable String language, @Nullable Integer timeOut, @Nullable Integer timeLimit, @Nullable Integer fixedLatency, @Nullable Integer gaussianLatency, Integer workerID) { - super(taskID, connection, queriesFile, timeOut, timeLimit, fixedLatency, gaussianLatency, workerID); - if (language != null) { - resultProcessor = new TypedFactory().create(language, new HashMap<>()); - } if (parameterName != null) { - parameter = parameterName; + this.parameter = parameterName; } if (responseType != null) { this.responseType = responseType; } } - void buildRequest(String query, String queryID) throws UnsupportedEncodingException { + void buildRequest(String query, String queryID) { String qEncoded = URLEncoder.encode(query, StandardCharsets.UTF_8); String addChar = "?"; - if (con.getEndpoint().contains("?")) { + if (this.con.getEndpoint().contains("?")) { addChar = "&"; } - String url = con.getEndpoint() + addChar + parameter + "=" + qEncoded; - request = new HttpGet(url); + String url = this.con.getEndpoint() + addChar + this.parameter + "=" + qEncoded; + this.request = new HttpGet(url); RequestConfig requestConfig = RequestConfig.custom() - .setSocketTimeout(timeOut.intValue()) - .setConnectTimeout(timeOut.intValue()) + .setSocketTimeout(this.timeOut.intValue()) + .setConnectTimeout(this.timeOut.intValue()) .build(); if (this.responseType != null) - request.setHeader(HttpHeaders.ACCEPT, this.responseType); + this.request.setHeader(HttpHeaders.ACCEPT, this.responseType); - request.setConfig(requestConfig); + this.request.setConfig(requestConfig); } } diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/impl/HttpPostWorker.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/impl/HttpPostWorker.java index dda40a889..f46a44afe 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/impl/HttpPostWorker.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/impl/HttpPostWorker.java @@ -10,6 +10,7 @@ import java.net.URLEncoder; import java.nio.charset.StandardCharsets; +import java.util.Map; /** * HTTP Post worker. @@ -23,11 +24,11 @@ public class HttpPostWorker extends HttpGetWorker { private String contentType = "text/plain"; + public HttpPostWorker(String taskID, Integer workerID, Connection connection, Map queries, @Nullable Integer timeLimit, @Nullable Integer timeOut, @Nullable Integer fixedLatency, @Nullable Integer gaussianLatency, @Nullable String parameterName, @Nullable String responseType, @Nullable String contentType) { + super(taskID, workerID, connection, queries, timeLimit, timeOut, fixedLatency, gaussianLatency, parameterName, responseType); - public HttpPostWorker(String taskID, Connection connection, String queriesFile, @Nullable String contentType, @Nullable String responseType, @Nullable String parameterName, @Nullable String language, @Nullable Integer timeOut, @Nullable Integer timeLimit, @Nullable Integer fixedLatency, @Nullable Integer gaussianLatency, Integer workerID) { - super(taskID, connection, queriesFile, responseType, parameterName, language, timeOut, timeLimit, fixedLatency, gaussianLatency, workerID); if (parameterName == null) { - parameter = null; + this.parameter = null; } if (contentType != null) { this.contentType = contentType; diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/impl/HttpWorker.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/impl/HttpWorker.java index 228b0f8cd..da6236d00 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/impl/HttpWorker.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/impl/HttpWorker.java @@ -1,11 +1,9 @@ package org.aksw.iguana.cc.worker.impl; import org.aksw.iguana.cc.config.elements.Connection; -import org.aksw.iguana.cc.lang.LanguageProcessor; -import org.aksw.iguana.cc.lang.impl.SPARQLLanguageProcessor; import org.aksw.iguana.cc.model.QueryExecutionStats; import org.aksw.iguana.cc.model.QueryResultHashKey; -import org.aksw.iguana.cc.worker.AbstractRandomQueryChooserWorker; +import org.aksw.iguana.cc.worker.AbstractWorker; import org.aksw.iguana.commons.annotation.Nullable; import org.aksw.iguana.commons.constants.COMMON; import org.apache.http.Header; @@ -28,6 +26,7 @@ import java.io.InputStream; import java.io.UnsupportedEncodingException; import java.time.Instant; +import java.util.Map; import java.util.concurrent.*; import static org.aksw.iguana.commons.time.TimeUtils.durationInMilliseconds; @@ -35,13 +34,12 @@ /** * Abstract HTTP worker */ -public abstract class HttpWorker extends AbstractRandomQueryChooserWorker { +public abstract class HttpWorker extends AbstractWorker { protected final ExecutorService resultProcessorService = Executors.newFixedThreadPool(5); protected ScheduledThreadPoolExecutor timeoutExecutorPool = new ScheduledThreadPoolExecutor(1); protected ConcurrentMap processedResults = new ConcurrentHashMap<>(); - protected LanguageProcessor resultProcessor = new SPARQLLanguageProcessor(); protected CloseableHttpClient client; protected HttpRequestBase request; protected ScheduledFuture abortCurrentRequestFuture; @@ -52,31 +50,31 @@ public abstract class HttpWorker extends AbstractRandomQueryChooserWorker { protected Instant requestStartTime; protected long tmpExecutedQueries = 0; - - public HttpWorker(String taskID, Connection connection, String queriesFile, @Nullable Integer timeOut, @Nullable Integer timeLimit, @Nullable Integer fixedLatency, @Nullable Integer gaussianLatency, Integer workerID) { - super(taskID, connection, queriesFile, timeOut, timeLimit, fixedLatency, gaussianLatency, workerID); - timeoutExecutorPool.setRemoveOnCancelPolicy(true); + public HttpWorker(String taskID, Integer workerID, Connection connection, Map queries, @Nullable Integer timeLimit, @Nullable Integer timeOut, @Nullable Integer fixedLatency, @Nullable Integer gaussianLatency) { + super(taskID, workerID, connection, queries, timeLimit, timeOut, fixedLatency, gaussianLatency); + this.timeoutExecutorPool.setRemoveOnCancelPolicy(true); } public ConcurrentMap getProcessedResults() { - return processedResults; + return this.processedResults; } protected void setTimeout(int timeOut) { - assert (request != null); - abortCurrentRequestFuture = timeoutExecutorPool.schedule( + assert (this.request != null); + this.abortCurrentRequestFuture = this.timeoutExecutorPool.schedule( () -> { synchronized (this) { - request.abort(); - requestTimedOut = true; + this.request.abort(); + this.requestTimedOut = true; } }, timeOut, TimeUnit.MILLISECONDS); } protected void abortTimeout() { - if (!abortCurrentRequestFuture.isDone()) - abortCurrentRequestFuture.cancel(false); + if (!this.abortCurrentRequestFuture.isDone()) { + this.abortCurrentRequestFuture.cancel(false); + } } @@ -85,12 +83,13 @@ public void stopSending() { super.stopSending(); abortTimeout(); try { - if (request != null && !request.isAborted()) - request.abort(); + if (this.request != null && !this.request.isAborted()) { + this.request.abort(); + } } catch (Exception ignored) { } closeClient(); - this.shutdownResultProcessor(); + shutdownResultProcessor(); } @@ -118,28 +117,28 @@ public void shutdownResultProcessor() { } synchronized protected void addResultsOnce(QueryExecutionStats queryExecutionStats) { - if (!resultsSaved) { - this.addResults(queryExecutionStats); - resultsSaved = true; + if (!this.resultsSaved) { + addResults(queryExecutionStats); + this.resultsSaved = true; } } @Override public void executeQuery(String query, String queryID) { - queryId = queryID; - resultsSaved = false; - requestTimedOut = false; + this.queryId = queryID; + this.resultsSaved = false; + this.requestTimedOut = false; - if (client == null) + if (this.client == null) initClient(); try { - buildRequest(query, queryId); + buildRequest(query, this.queryId); - setTimeout(timeOut.intValue()); - - requestStartTime = Instant.now(); - response = client.execute(request, getAuthContext(con.getEndpoint())); + setTimeout(this.timeOut.intValue()); + + this.requestStartTime = Instant.now(); + this.response = this.client.execute(this.request, getAuthContext(this.con.getEndpoint())); // method to process the result in background processHttpResponse(); @@ -148,10 +147,10 @@ public void executeQuery(String query, String queryID) { } catch (ClientProtocolException e) { handleException(query, COMMON.QUERY_HTTP_FAILURE, e); } catch (IOException e) { - if (requestTimedOut) { + if (this.requestTimedOut) { LOGGER.warn("Worker[{} : {}]: Reached timeout on query (ID {})\n{}", - this.workerType, this.workerID, queryId, query); - addResultsOnce(new QueryExecutionStats(queryId, COMMON.QUERY_SOCKET_TIMEOUT, timeOut)); + this.workerType, this.workerID, this.queryId, query); + addResultsOnce(new QueryExecutionStats(this.queryId, COMMON.QUERY_SOCKET_TIMEOUT, this.timeOut)); } else { handleException(query, COMMON.QUERY_UNKNOWN_EXCEPTION, e); } @@ -164,84 +163,84 @@ public void executeQuery(String query, String queryID) { } private void handleException(String query, Long cause, Exception e) { - double duration = durationInMilliseconds(requestStartTime, Instant.now()); - addResultsOnce(new QueryExecutionStats(queryId, cause, duration)); + double duration = durationInMilliseconds(this.requestStartTime, Instant.now()); + addResultsOnce(new QueryExecutionStats(this.queryId, cause, duration)); LOGGER.warn("Worker[{} : {}]: {} on query (ID {})\n{}", - this.workerType, this.workerID, e.getMessage(), queryId, query); + this.workerType, this.workerID, e.getMessage(), this.queryId, query); closeClient(); initClient(); } protected void processHttpResponse() { - int responseCode = response.getStatusLine().getStatusCode(); + int responseCode = this.response.getStatusLine().getStatusCode(); boolean responseCodeSuccess = responseCode >= 200 && responseCode < 300; boolean responseCodeOK = responseCode == 200; if (responseCodeOK) { // response status is OK (200) // get content type header - HttpEntity httpResponse = response.getEntity(); + HttpEntity httpResponse = this.response.getEntity(); Header contentTypeHeader = new BasicHeader(httpResponse.getContentType().getName(), httpResponse.getContentType().getValue()); // get content stream try (InputStream contentStream = httpResponse.getContent()) { // read content stream with resultProcessor, return length, set string in StringBuilder. ByteArrayOutputStream responseBody = new ByteArrayOutputStream(); - long length = resultProcessor.readResponse(contentStream, responseBody); - tmpExecutedQueries++; + long length = this.queryHandler.getLanguageProcessor().readResponse(contentStream, responseBody); + this.tmpExecutedQueries++; // check if such a result was already parsed and is cached - double duration = durationInMilliseconds(requestStartTime, Instant.now()); + double duration = durationInMilliseconds(this.requestStartTime, Instant.now()); synchronized (this) { QueryResultHashKey resultCacheKey = new QueryResultHashKey(queryId, length); - if (processedResults.containsKey(resultCacheKey)) { + if (this.processedResults.containsKey(resultCacheKey)) { LOGGER.debug("found result cache key {} ", resultCacheKey); - Long preCalculatedResultSize = processedResults.get(resultCacheKey); - addResultsOnce(new QueryExecutionStats(queryId, COMMON.QUERY_SUCCESS, duration, preCalculatedResultSize)); + Long preCalculatedResultSize = this.processedResults.get(resultCacheKey); + addResultsOnce(new QueryExecutionStats(this.queryId, COMMON.QUERY_SUCCESS, duration, preCalculatedResultSize)); } else { // otherwise: parse it. The parsing result is cached for the next time. if (!this.endSignal) { - resultProcessorService.submit(new HttpResultProcessor(this, queryId, duration, contentTypeHeader, responseBody, length)); - resultsSaved = true; + this.resultProcessorService.submit(new HttpResultProcessor(this, this.queryId, duration, contentTypeHeader, responseBody, length)); + this.resultsSaved = true; } } } } catch (IOException | TimeoutException e) { - double duration = durationInMilliseconds(requestStartTime, Instant.now()); - addResultsOnce(new QueryExecutionStats(queryId, COMMON.QUERY_HTTP_FAILURE, duration)); + double duration = durationInMilliseconds(this.requestStartTime, Instant.now()); + addResultsOnce(new QueryExecutionStats(this.queryId, COMMON.QUERY_HTTP_FAILURE, duration)); } } else if (responseCodeSuccess) { // response status is succeeded (2xx) but not OK (200) - double duration = durationInMilliseconds(requestStartTime, Instant.now()); - addResultsOnce(new QueryExecutionStats(queryId, COMMON.QUERY_SUCCESS, duration, 0)); + double duration = durationInMilliseconds(this.requestStartTime, Instant.now()); + addResultsOnce(new QueryExecutionStats(this.queryId, COMMON.QUERY_SUCCESS, duration, 0)); } else { // response status indicates that the query did not succeed (!= 2xx) - double duration = durationInMilliseconds(requestStartTime, Instant.now()); - addResultsOnce(new QueryExecutionStats(queryId, COMMON.QUERY_HTTP_FAILURE, duration)); + double duration = durationInMilliseconds(this.requestStartTime, Instant.now()); + addResultsOnce(new QueryExecutionStats(this.queryId, COMMON.QUERY_HTTP_FAILURE, duration)); } } abstract void buildRequest(String query, String queryID) throws UnsupportedEncodingException; protected void initClient() { - client = HttpClients.custom().setConnectionManager(new BasicHttpClientConnectionManager()).build(); + this.client = HttpClients.custom().setConnectionManager(new BasicHttpClientConnectionManager()).build(); } protected void closeClient() { closeResponse(); try { - if (client != null) - client.close(); + if (this.client != null) + this.client.close(); } catch (IOException e) { LOGGER.error("Could not close http response ", e); } - client = null; + this.client = null; } protected void closeResponse() { try { - if (response != null) - response.close(); + if (this.response != null) + this.response.close(); } catch (IOException e) { LOGGER.error("Could not close Client ", e); } - response = null; + this.response = null; } /** @@ -256,8 +255,8 @@ static class HttpResultProcessor implements Runnable { private final String queryId; private final double duration; private final Header contentTypeHeader; - private ByteArrayOutputStream contentStream; private final long contentLength; + private ByteArrayOutputStream contentStream; public HttpResultProcessor(HttpWorker httpWorker, String queryId, double duration, Header contentTypeHeader, ByteArrayOutputStream contentStream, long contentLength) { this.httpWorker = httpWorker; @@ -272,22 +271,22 @@ public HttpResultProcessor(HttpWorker httpWorker, String queryId, double duratio public void run() { // Result size is not saved before. Process the http response. - ConcurrentMap processedResults = httpWorker.getProcessedResults(); - QueryResultHashKey resultCacheKey = new QueryResultHashKey(queryId, contentLength); + ConcurrentMap processedResults = this.httpWorker.getProcessedResults(); + QueryResultHashKey resultCacheKey = new QueryResultHashKey(this.queryId, this.contentLength); try { //String content = contentStream.toString(StandardCharsets.UTF_8.name()); //contentStream = null; // might be hugh, dereference immediately after consumed - Long resultSize = httpWorker.resultProcessor.getResultSize(contentTypeHeader, contentStream, contentLength); - contentStream = null; + Long resultSize = this.httpWorker.queryHandler.getLanguageProcessor().getResultSize(this.contentTypeHeader, this.contentStream, this.contentLength); + this.contentStream = null; // Save the result size to be re-used processedResults.put(resultCacheKey, resultSize); LOGGER.debug("added Result Cache Key {}", resultCacheKey); - httpWorker.addResults(new QueryExecutionStats(queryId, COMMON.QUERY_SUCCESS, duration, resultSize)); + this.httpWorker.addResults(new QueryExecutionStats(this.queryId, COMMON.QUERY_SUCCESS, this.duration, resultSize)); } catch (IOException | ParseException | ParserConfigurationException | SAXException e) { LOGGER.error("Query results could not be parsed. ", e); - httpWorker.addResults(new QueryExecutionStats(queryId, COMMON.QUERY_UNKNOWN_EXCEPTION, duration)); + this.httpWorker.addResults(new QueryExecutionStats(this.queryId, COMMON.QUERY_UNKNOWN_EXCEPTION, this.duration)); } catch (Exception e) { e.printStackTrace(); } diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/impl/SPARQLWorker.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/impl/SPARQLWorker.java index 604b8be0a..6c6a26e66 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/impl/SPARQLWorker.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/impl/SPARQLWorker.java @@ -4,6 +4,7 @@ import org.aksw.iguana.commons.annotation.Nullable; import org.aksw.iguana.commons.annotation.Shorthand; +import java.util.Map; /** @@ -14,8 +15,8 @@ @Shorthand("SPARQLWorker") public class SPARQLWorker extends HttpGetWorker { - public SPARQLWorker(String taskID, Connection connection, String queriesFile, @Nullable String responseType, @Nullable String parameterName, @Nullable Integer timeOut, @Nullable Integer timeLimit, @Nullable Integer fixedLatency, @Nullable Integer gaussianLatency, Integer workerID) { - super(taskID, connection, queriesFile, responseType, parameterName, "lang.SPARQL", timeOut, timeLimit, fixedLatency, gaussianLatency, workerID); + public SPARQLWorker(String taskID, Integer workerID, Connection connection, Map queries, @Nullable Integer timeLimit, @Nullable Integer timeOut, @Nullable Integer fixedLatency, @Nullable Integer gaussianLatency, @Nullable String parameterName, @Nullable String responseType) { + super(taskID, workerID, connection, queries, timeLimit, timeOut, fixedLatency, gaussianLatency, parameterName, responseType); } } diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/impl/UPDATEWorker.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/impl/UPDATEWorker.java index f469712d9..c78b821d2 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/impl/UPDATEWorker.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/impl/UPDATEWorker.java @@ -2,7 +2,6 @@ import org.aksw.iguana.cc.config.elements.Connection; import org.aksw.iguana.cc.model.QueryExecutionStats; -import org.aksw.iguana.cc.query.set.QuerySet; import org.aksw.iguana.cc.worker.impl.update.UpdateTimer; import org.aksw.iguana.commons.annotation.Nullable; import org.aksw.iguana.commons.annotation.Shorthand; @@ -10,132 +9,115 @@ import java.io.IOException; import java.time.Instant; +import java.util.Map; import java.util.Properties; import static org.aksw.iguana.commons.time.TimeUtils.durationInMilliseconds; /** - * * A Worker using SPARQL Updates to create service request. - * - * @author f.conrads * + * @author f.conrads */ @Shorthand("UPDATEWorker") public class UPDATEWorker extends HttpPostWorker { - - private int currentQueryID = 0; - private UpdateTimer updateTimer = new UpdateTimer(); - private final String timerStrategy; - - public UPDATEWorker(String taskID, Connection connection, String queriesFile, @Nullable String timerStrategy, @Nullable Integer timeOut, @Nullable Integer timeLimit, @Nullable Integer fixedLatency, @Nullable Integer gaussianLatency, Integer workerID) { - super(taskID, connection, queriesFile, "application/sparql-update", null, null, "lang.SPARQL", timeOut, timeLimit, fixedLatency, gaussianLatency, workerID); - this.timerStrategy = timerStrategy; - } - - @Override - public void startWorker(){ - setUpdateTimer(this.timerStrategy); - super.startWorker(); - } - - @Override - public void waitTimeMs() { - double wait = this.updateTimer.calculateTime(durationInMilliseconds(startTime, Instant.now()), this.tmpExecutedQueries); - LOGGER.debug("Worker[{{}} : {{}}]: Time to wait for next Query {{}}", workerType, workerID, wait); - try { - Thread.sleep((long)wait); - } catch (InterruptedException e) { - LOGGER.error("Worker[{{}} : {{}}]: Could not wait time before next query due to: {{}}", workerType, - workerID, e); - LOGGER.error("", e); - } - super.waitTimeMs(); - } - - @Override - public synchronized void addResults(QueryExecutionStats results) - { - // create Properties store it in List - Properties result = new Properties(); - result.setProperty(COMMON.EXPERIMENT_TASK_ID_KEY, this.taskID); - result.put(COMMON.RECEIVE_DATA_TIME, results.getExecutionTime()); - result.put(COMMON.RECEIVE_DATA_SUCCESS, results.getResponseCode()); - result.put(COMMON.RECEIVE_DATA_SIZE, results.getResultSize()); - result.put(COMMON.QUERY_HASH, queryHash); - result.setProperty(COMMON.QUERY_ID_KEY, results.getQueryID()); - result.put(COMMON.PENALTY, this.timeOut); - // Add extra Meta Key, worker ID and worker Type - result.put(COMMON.EXTRA_META_KEY, this.extra); - setResults(result); - executedQueries++; - - - } - - @Override - public void getNextQuery(StringBuilder queryStr, StringBuilder queryID) throws IOException { - // If there is no more update send end signal, as their is nothing to do anymore - if (this.currentQueryID >= this.queryFileList.length) { - this.stopSending(); - return; - } - // get next Query File and next random Query out of it. - QuerySet currentQueryFile = this.queryFileList[this.currentQueryID++]; - queryID.append(currentQueryFile.getName()); - - queryStr.append(currentQueryFile.getQueryAtPos(0)); - - } - - @Override - public void setQueriesList(QuerySet[] updateFiles) { - super.setQueriesList(updateFiles); - } - - /** - * Sets Update Timer according to strategy - * - * @param strategyStr - * The String representation of a UpdateTimer.Strategy - */ - private void setUpdateTimer(String strategyStr) { - if (strategyStr == null) - return; - UpdateTimer.Strategy strategy = UpdateTimer.Strategy.valueOf(strategyStr.toUpperCase()); - switch (strategy) { - case FIXED: - if (timeLimit != null) { - this.updateTimer = new UpdateTimer(this.timeLimit/this.queryFileList.length); - } else { - LOGGER.warn("Worker[{{}} : {{}}]: FIXED Updates can only be used with timeLimit!", workerType, - workerID); - } - break; - case DISTRIBUTED: - if (timeLimit != null) { - this.updateTimer = new UpdateTimer(this.queryFileList.length, this.timeLimit); - } else { - LOGGER.warn("Worker[{{}} : {{}}]: DISTRIBUTED Updates can only be used with timeLimit!", workerType, - workerID); - } - break; - default: - break; - } - LOGGER.debug("Worker[{{}} : {{}}]: UpdateTimer was set to UpdateTimer:{{}}", workerType, workerID, updateTimer); - } - - - - /** - * Checks if one queryMix was already executed, as it does not matter how many mixes should be executed - * @param noOfQueryMixes - * @return - */ - @Override - public boolean hasExecutedNoOfQueryMixes(Long noOfQueryMixes){ - return getExecutedQueries() / (getNoOfQueries() * 1.0) >= 1; - } + private final String timerStrategy; + private UpdateTimer updateTimer = new UpdateTimer(); + + private int queryCount; + + public UPDATEWorker(String taskID, Integer workerID, Connection connection, Map queries, @Nullable Integer timeLimit, @Nullable Integer timeOut, @Nullable Integer fixedLatency, @Nullable Integer gaussianLatency, @Nullable String timerStrategy) { + super(taskID, workerID, connection, queries, timeLimit, timeOut, fixedLatency, gaussianLatency, null, null, "application/sparql-update"); + this.timerStrategy = timerStrategy; + } + + @Override + public void startWorker() { + setUpdateTimer(this.timerStrategy); + super.startWorker(); + } + + @Override + public void waitTimeMs() { + double wait = this.updateTimer.calculateTime(durationInMilliseconds(this.startTime, Instant.now()), this.tmpExecutedQueries); + LOGGER.debug("Worker[{{}} : {{}}]: Time to wait for next Query {{}}", this.workerType, this.workerID, wait); + try { + Thread.sleep((long) wait); + } catch (InterruptedException e) { + LOGGER.error("Worker[{{}} : {{}}]: Could not wait time before next query due to: {{}}", this.workerType, this.workerID, e); + LOGGER.error("", e); + } + super.waitTimeMs(); + } + + @Override + public synchronized void addResults(QueryExecutionStats results) { + // create Properties store it in List + Properties result = new Properties(); + result.setProperty(COMMON.EXPERIMENT_TASK_ID_KEY, this.taskID); + result.put(COMMON.RECEIVE_DATA_TIME, results.getExecutionTime()); + result.put(COMMON.RECEIVE_DATA_SUCCESS, results.getResponseCode()); + result.put(COMMON.RECEIVE_DATA_SIZE, results.getResultSize()); + result.put(COMMON.QUERY_HASH, this.queryHash); + result.setProperty(COMMON.QUERY_ID_KEY, results.getQueryID()); + result.put(COMMON.PENALTY, this.timeOut); + // Add extra Meta Key, worker ID and worker Type + result.put(COMMON.EXTRA_META_KEY, this.extra); + setResults(result); + this.executedQueries++; + } + + @Override + public void getNextQuery(StringBuilder queryStr, StringBuilder queryID) throws IOException { + // If there is no more update send end signal, as there is nothing to do anymore + if (this.queryCount >= this.queryHandler.getQueryCount()) { + stopSending(); + return; + } + + this.queryHandler.getNextQuery(queryStr, queryID); + this.queryCount++; + } + + /** + * Sets Update Timer according to strategy + * + * @param strategyStr The String representation of a UpdateTimer.Strategy + */ + private void setUpdateTimer(String strategyStr) { + if (strategyStr == null) return; + UpdateTimer.Strategy strategy = UpdateTimer.Strategy.valueOf(strategyStr.toUpperCase()); + switch (strategy) { + case FIXED: + if (this.timeLimit != null) { + this.updateTimer = new UpdateTimer(this.timeLimit / this.queryHandler.getQueryCount()); + } else { + LOGGER.warn("Worker[{{}} : {{}}]: FIXED Updates can only be used with timeLimit!", this.workerType, this.workerID); + } + break; + case DISTRIBUTED: + if (this.timeLimit != null) { + this.updateTimer = new UpdateTimer(this.queryHandler.getQueryCount(), this.timeLimit); + } else { + LOGGER.warn("Worker[{{}} : {{}}]: DISTRIBUTED Updates can only be used with timeLimit!", this.workerType, this.workerID); + } + break; + default: + break; + } + LOGGER.debug("Worker[{{}} : {{}}]: UpdateTimer was set to UpdateTimer:{{}}", this.workerType, this.workerID, this.updateTimer); + } + + + /** + * Checks if one queryMix was already executed, as it does not matter how many mixes should be executed + * + * @param noOfQueryMixes + * @return + */ + @Override + public boolean hasExecutedNoOfQueryMixes(Long noOfQueryMixes) { + return getExecutedQueries() / (getNoOfQueries() * 1.0) >= 1; + } } diff --git a/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/impl/DelimInstancesQueryHandlerTest.java b/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/impl/DelimInstancesQueryHandlerTest.java deleted file mode 100644 index b84d03af9..000000000 --- a/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/impl/DelimInstancesQueryHandlerTest.java +++ /dev/null @@ -1,120 +0,0 @@ -package org.aksw.iguana.cc.query.impl; - -import org.aksw.iguana.cc.config.elements.Connection; -import org.aksw.iguana.cc.query.set.QuerySet; -import org.aksw.iguana.cc.worker.Worker; -import org.aksw.iguana.cc.worker.impl.SPARQLWorker; -import org.aksw.iguana.cc.worker.impl.UPDATEWorker; -import org.apache.commons.io.FileUtils; -import org.apache.jena.ext.com.google.common.collect.Lists; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; - -import java.io.File; -import java.io.IOException; -import java.io.PrintWriter; -import java.util.*; - -import static org.junit.Assert.assertEquals; - -@RunWith(Parameterized.class) -public class DelimInstancesQueryHandlerTest { - - private final boolean isUpdate; - private final String delim; - private String[] queryStr; - private String dir = UUID.randomUUID().toString(); - private File queriesFile; - - @Parameterized.Parameters - public static Collection data(){ - Collection testData = new ArrayList(); - testData.add(new Object[]{new String[]{"SELECT * \n{\n?s ?p ?o\n}", "doesn't matter", "as long as they are not empty", "the only thing which won't do is the triplestats"}, false, ""}); - testData.add(new Object[]{new String[]{"SELECT * {?s ?p ?o}", "doesn't matter", "", "the only thing \nwhich won't do is the triplestats"}, false, ""}); - testData.add(new Object[]{new String[]{"UPDATE * \n{?s ?p ?o}", "UPDATE \ndoesn't matter", "", "UPDATE\n the only thing which won't do is the triplestats"}, true, ""}); - testData.add(new Object[]{new String[]{"SELECT * \n{\n?s ?p ?o\n}", "doesn't matter", "as long as they are not empty", "the only thing which won't do is the triplestats"}, false, "###"}); - testData.add(new Object[]{new String[]{"SELECT * {?s ?p ?o}", "doesn't matter", "", "the only thing \n\nwhich won't do is the triplestats"}, false, "###"}); - testData.add(new Object[]{new String[]{"UPDATE * \n{?s ?p ?o}", "UPDATE \ndoesn't matter", "", "UPDATE\n\n the only thing which won't do is the triplestats"}, true, "###"}); - - return testData; - } - - public DelimInstancesQueryHandlerTest(String[] queryStr, boolean isUpdate, String delim){ - this.queryStr = queryStr; - this.isUpdate=isUpdate; - this.delim=delim; - } - - @Before - public void createFolder() throws IOException { - //File f = new File(this.dir); - //f.mkdir(); - String queryFile = UUID.randomUUID().toString(); - File f = new File(queryFile); - f.createNewFile(); - try(PrintWriter pw = new PrintWriter(f)){ - for(String query : queryStr) { - pw.println(query); - pw.println(delim); - } - } - //remove empty lines after printing them, so the expected asserts will correctly assume that the empty limes are ignored - List tmpList = Lists.newArrayList(queryStr); - Iterator it = tmpList.iterator(); - while(it.hasNext()){ - if(it.next().isEmpty()){ - it.remove(); - } - } - this.queryStr= tmpList.toArray(new String[]{}); - this.queriesFile = f; - f.deleteOnExit(); - } - - @After - public void removeFolder() throws IOException { - File f = new File(this.dir); - FileUtils.deleteDirectory(f); - } - - @Test - public void testQueryCreation() throws IOException { - //Get queries file - Connection con = new Connection(); - con.setName("a"); - con.setEndpoint("http://test.com"); - Worker worker = getWorker(con, 1, "1"); - DelimInstancesQueryHandler qh = new DelimInstancesQueryHandler(delim, Lists.newArrayList(worker)); - qh.setOutputFolder(this.dir); - Map map = qh.generate(); - List expected = new ArrayList(); - List actual = new ArrayList(); - - for(String qStr : queryStr){ - expected.add(qStr); - } - - for(QuerySet querySet : map.get(this.queriesFile.getAbsolutePath())){ - assertEquals(1, querySet.size()); - actual.add(querySet.getQueryAtPos(0)); - } - assertEquals(expected.size(), actual.size()); - actual.removeAll(expected); - assertEquals(0, actual.size()); - assertEquals(queryStr.length, map.get(this.queriesFile.getAbsolutePath()).length); - } - - - public Worker getWorker(Connection con, int id, String taskID){ - if(isUpdate){ - return new UPDATEWorker(taskID, con, this.queriesFile.getAbsolutePath(), null, null, null, null,null, id); - } - else { - return new SPARQLWorker(taskID, con, this.queriesFile.getAbsolutePath(), null, null, null, null, null, null, id); - } - } - -} diff --git a/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/impl/InstancesQueryHandlerTest.java b/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/impl/InstancesQueryHandlerTest.java deleted file mode 100644 index 33f812245..000000000 --- a/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/impl/InstancesQueryHandlerTest.java +++ /dev/null @@ -1,118 +0,0 @@ -package org.aksw.iguana.cc.query.impl; - -import org.aksw.iguana.cc.config.elements.Connection; -import org.aksw.iguana.cc.query.set.QuerySet; -import org.aksw.iguana.cc.query.set.impl.FileBasedQuerySet; -import org.aksw.iguana.cc.worker.Worker; -import org.aksw.iguana.cc.worker.impl.SPARQLWorker; -import org.aksw.iguana.cc.worker.impl.UPDATEWorker; -import org.apache.commons.io.FileUtils; -import org.apache.jena.ext.com.google.common.collect.Lists; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; - -import java.io.File; -import java.io.IOException; -import java.io.PrintWriter; -import java.util.*; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -@RunWith(Parameterized.class) -public class InstancesQueryHandlerTest { - - private final boolean isUpdate; - private String[] queryStr; - private String dir = UUID.randomUUID().toString(); - private File queriesFile; - - @Parameterized.Parameters - public static Collection data(){ - Collection testData = new ArrayList(); - testData.add(new Object[]{new String[]{"SELECT * {?s ?p ?o}", "doesn't matter", "as long as they are not empty", "the only thing which won't do is the triplestats"}, false}); - testData.add(new Object[]{new String[]{"SELECT * {?s ?p ?o}", "doesn't matter", "", "the only thing which won't do is the triplestats"}, false}); - testData.add(new Object[]{new String[]{"UPDATE * {?s ?p ?o}", "UPDATE doesn't matter", "", "UPDATE the only thing which won't do is the triplestats"}, true}); - - return testData; - } - - public InstancesQueryHandlerTest(String[] queryStr, boolean isUpdate){ - this.queryStr = queryStr; - this.isUpdate=isUpdate; - } - - @Before - public void createFolder() throws IOException { - //File f = new File(this.dir); - //f.mkdir(); - String queryFile = UUID.randomUUID().toString(); - File f = new File(queryFile); - f.createNewFile(); - try(PrintWriter pw = new PrintWriter(f)){ - for(String query : queryStr) { - pw.println(query); - } - } - //remove empty lines after printing them, so the expected asserts will correctly assume that the empty limes are ignored - List tmpList = Lists.newArrayList(queryStr); - Iterator it = tmpList.iterator(); - while(it.hasNext()){ - if(it.next().isEmpty()){ - it.remove(); - } - } - this.queryStr= tmpList.toArray(new String[]{}); - this.queriesFile = f; - f.deleteOnExit(); - } - - @After - public void removeFolder() throws IOException { - File f = new File(this.dir); - FileUtils.deleteDirectory(f); - } - - - @Test - public void testQueryCreation() throws IOException { - //Get queries file - Connection con = new Connection(); - con.setName("a"); - con.setEndpoint("http://test.com"); - Worker worker = getWorker(con, 1, "1"); - InstancesQueryHandler qh = new InstancesQueryHandler(Lists.newArrayList(worker)); - qh.setOutputFolder(this.dir); - Map map = qh.generate(); - List expected = new ArrayList(); - List actual = new ArrayList(); - - for(String qStr : queryStr){ - expected.add(qStr); - } - - for(QuerySet querySet : map.get(this.queriesFile.getAbsolutePath())){ - assertEquals(1, querySet.size()); - actual.add(querySet.getQueryAtPos(0)); - } - assertEquals(expected.size(), actual.size()); - actual.removeAll(expected); - assertEquals(actual.size(),0); - assertEquals(queryStr.length, map.get(this.queriesFile.getAbsolutePath()).length); - } - - - public Worker getWorker(Connection con, int id, String taskID){ - if(isUpdate){ - return new UPDATEWorker(taskID, con, this.queriesFile.getAbsolutePath(), null, null, null, null,null, id); - } - else { - return new SPARQLWorker(taskID, con, this.queriesFile.getAbsolutePath(), null, null, null, null, null, null, id); - } - } - - -} diff --git a/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/impl/UpdatePathTest.java b/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/impl/UpdatePathTest.java deleted file mode 100644 index ced92dc0e..000000000 --- a/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/impl/UpdatePathTest.java +++ /dev/null @@ -1,46 +0,0 @@ -package org.aksw.iguana.cc.query.impl; - -import org.aksw.iguana.cc.query.set.QuerySet; -import org.aksw.iguana.cc.query.set.impl.FileBasedQuerySet; -import org.apache.jena.ext.com.google.common.collect.Lists; -import org.aksw.iguana.cc.config.elements.Connection; -import org.aksw.iguana.cc.worker.Worker; -import org.aksw.iguana.cc.worker.impl.UPDATEWorker; -import org.junit.Test; - -import java.io.File; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -public class UpdatePathTest { - - @Test - public void checkUpdatePath(){ - Connection con = new Connection(); - con.setName("a"); - con.setEndpoint("http://test.com"); - String updateDir = "src/test/resources/updates/"; - Worker worker = new UPDATEWorker("1", con, updateDir, null, null, null, null,null, 1); - - InstancesQueryHandler qh = new InstancesQueryHandler(Lists.newArrayList(worker)); - Map map = qh.generate(); - assertEquals(1, map.size()); - QuerySet[] updates = map.get(updateDir); - assertEquals(2, updates.length); - List paths = new ArrayList(); - for(File f: new File(updateDir).listFiles()){ - paths.add(f.getAbsolutePath()); - } - assertEquals(2, paths.size()); - for(QuerySet actual : updates){ - assertTrue(actual instanceof FileBasedQuerySet); - paths.remove(((FileBasedQuerySet)actual).getFile().getAbsolutePath()); - } - assertEquals(0, paths.size()); - } - -} diff --git a/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/worker/HTTPWorkerTest.java b/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/worker/HTTPWorkerTest.java index 6fd7f8d08..b9329d8ad 100644 --- a/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/worker/HTTPWorkerTest.java +++ b/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/worker/HTTPWorkerTest.java @@ -2,13 +2,11 @@ import org.aksw.iguana.cc.config.elements.Connection; import org.aksw.iguana.cc.lang.impl.SPARQLLanguageProcessor; -import org.aksw.iguana.cc.query.impl.InstancesQueryHandler; import org.aksw.iguana.cc.utils.FileUtils; import org.aksw.iguana.cc.worker.impl.HttpGetWorker; import org.aksw.iguana.cc.worker.impl.HttpPostWorker; import org.aksw.iguana.cc.worker.impl.HttpWorker; import org.aksw.iguana.commons.constants.COMMON; -import org.apache.jena.ext.com.google.common.collect.Lists; import org.junit.*; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; @@ -19,10 +17,7 @@ import java.io.IOException; import java.net.InetSocketAddress; import java.net.SocketAddress; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Properties; -import java.util.UUID; +import java.util.*; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -32,47 +27,64 @@ public class HTTPWorkerTest { private static final int FAST_SERVER_PORT = 8025; - private final String service; - private static WorkerServerMock fastServerContainer; private static ContainerServer fastServer; private static SocketConnection fastConnection; + private final String service; private final Boolean isPost; - - private String queriesFile="src/test/resources/workers/single-query.txt"; - private String responseType; - private String parameter; - private String query; - private String queryID; - private boolean isFail; + private final HashMap queries; + + private final String queriesFile = "src/test/resources/workers/single-query.txt"; + private final String responseType; + private final String parameter; + private final String query; + private final String queryID; + private final boolean isFail; private String outputDir; - private Integer fixedLatency; - private Integer gaussianLatency; + private final Integer fixedLatency; + private final Integer gaussianLatency; + + public HTTPWorkerTest(String query, String queryID, String responseType, String parameter, Integer fixedLatency, Integer gaussianLatency, Boolean isFail, Boolean isPost) { + this.query = query; + this.queryID = queryID; + this.responseType = responseType; + this.parameter = parameter; + this.isFail = isFail; + this.isPost = isPost; + this.fixedLatency = fixedLatency; + this.gaussianLatency = gaussianLatency; + this.service = "http://localhost:8025"; + + this.queries = new HashMap<>(); + this.queries.put("location", this.queriesFile); + + //warmup + getWorker("1").executeQuery("test", "test"); + } @Parameterized.Parameters - public static Collection data(){ - Collection testData = new ArrayList(); + public static Collection data() { + Collection testData = new ArrayList<>(); //get tests - testData.add(new Object[]{"Random Text", "doc1", "text/plain", "text", 100,50, false, false}); - testData.add(new Object[]{UUID.randomUUID().toString(), UUID.randomUUID().toString(), "text/plain", "text", 100,50, false, false}); + testData.add(new Object[]{"Random Text", "doc1", "text/plain", "text", 100, 50, false, false}); + testData.add(new Object[]{UUID.randomUUID().toString(), UUID.randomUUID().toString(), "text/plain", "text", 100, 50, false, false}); - testData.add(new Object[]{"Random Text", "doc1", "text/plain", "test", 100,50, true, false}); - testData.add(new Object[]{"Random Text", "doc1", null, "text", 100,50, false, false}); + testData.add(new Object[]{"Random Text", "doc1", "text/plain", "test", 100, 50, true, false}); + testData.add(new Object[]{"Random Text", "doc1", null, "text", 100, 50, false, false}); //post tests - testData.add(new Object[]{"Random Text", "doc1", "text/plain", "text", 100,50, false, true}); - testData.add(new Object[]{UUID.randomUUID().toString(), UUID.randomUUID().toString(), "text/plain", "text", 100,50, false, true}); + testData.add(new Object[]{"Random Text", "doc1", "text/plain", "text", 100, 50, false, true}); + testData.add(new Object[]{UUID.randomUUID().toString(), UUID.randomUUID().toString(), "text/plain", "text", 100, 50, false, true}); - testData.add(new Object[]{"Random Text", "doc1", "text/plain", "test", 100,50, true, true}); - testData.add(new Object[]{"Random Text", "doc1", "text/plain", null, 100,50, true, true}); - testData.add(new Object[]{"Random Text", "doc1", null, "text", 100,50, false, true}); + testData.add(new Object[]{"Random Text", "doc1", "text/plain", "test", 100, 50, true, true}); + testData.add(new Object[]{"Random Text", "doc1", "text/plain", null, 100, 50, true, true}); + testData.add(new Object[]{"Random Text", "doc1", null, "text", 100, 50, false, true}); return testData; } @BeforeClass public static void startServer() throws IOException { - fastServerContainer = new WorkerServerMock(); - fastServer = new ContainerServer(fastServerContainer); + fastServer = new ContainerServer(new WorkerServerMock()); fastConnection = new SocketConnection(fastServer); SocketAddress address1 = new InetSocketAddress(FAST_SERVER_PORT); fastConnection.connect(address1); @@ -85,39 +97,24 @@ public static void stopServer() throws IOException { fastServer.stop(); } - - public HTTPWorkerTest(String query, String queryID, String responseType, String parameter, Integer fixedLatency, Integer gaussianLatency, Boolean isFail, Boolean isPost){ - this.query=query; - this.queryID=queryID; - this.responseType=responseType; - this.parameter=parameter; - this.isFail=isFail; - this.isPost=isPost; - this.fixedLatency=fixedLatency; - this.gaussianLatency=gaussianLatency; - this.service = "http://localhost:8025"; - //warmup - getWorker("1").executeQuery("test", "test"); - } - @Before - public void setOutputDir(){ + public void setOutputDir() { this.outputDir = UUID.randomUUID().toString(); } @After public void deleteFolder() throws IOException { - org.apache.commons.io.FileUtils.deleteDirectory(new File(outputDir)); + org.apache.commons.io.FileUtils.deleteDirectory(new File(this.outputDir)); } @Test - public void testExecution() throws InterruptedException, IOException { + public void testExecution() throws InterruptedException { // check if correct param name was set - String taskID="123/1/1/"; + String taskID = "123/1/1/"; HttpWorker getWorker = getWorker(taskID); - getWorker.executeQuery(query, queryID); + getWorker.executeQuery(this.query, this.queryID); //as the result processing is in the background we have to wait for it. Thread.sleep(1000); Collection results = getWorker.popQueryResults(); @@ -126,26 +123,24 @@ public void testExecution() throws InterruptedException, IOException { assertEquals(taskID, p.get(COMMON.EXPERIMENT_TASK_ID_KEY)); - assertEquals(queryID, p.get(COMMON.QUERY_ID_KEY)); + assertEquals(this.queryID, p.get(COMMON.QUERY_ID_KEY)); assertEquals(180000.0, p.get(COMMON.PENALTY)); - assertTrue(((Properties)p.get(COMMON.EXTRA_META_KEY)).isEmpty()); - if(isPost){ + assertTrue(((Properties) p.get(COMMON.EXTRA_META_KEY)).isEmpty()); + if (isPost) { assertEquals(200.0, (double) p.get(COMMON.RECEIVE_DATA_TIME), 20.0); - } - else { + } else { assertEquals(100.0, (double) p.get(COMMON.RECEIVE_DATA_TIME), 20.0); } - if(isFail){ - assertEquals(-2l, p.get(COMMON.RECEIVE_DATA_SUCCESS)); - assertEquals(0l, p.get(COMMON.RECEIVE_DATA_SIZE)); - } - else{ - assertEquals(1l, p.get(COMMON.RECEIVE_DATA_SUCCESS)); - if(responseType!= null && responseType.equals("text/plain")) { - assertEquals(4l, p.get(COMMON.RECEIVE_DATA_SIZE)); + if (isFail) { + assertEquals(-2L, p.get(COMMON.RECEIVE_DATA_SUCCESS)); + assertEquals(0L, p.get(COMMON.RECEIVE_DATA_SIZE)); + } else { + assertEquals(1L, p.get(COMMON.RECEIVE_DATA_SUCCESS)); + if (this.responseType != null && this.responseType.equals("text/plain")) { + assertEquals(4L, p.get(COMMON.RECEIVE_DATA_SIZE)); } - if(responseType==null || responseType.equals(SPARQLLanguageProcessor.QUERY_RESULT_TYPE_JSON)){ - assertEquals(2l, p.get(COMMON.RECEIVE_DATA_SIZE)); + if (this.responseType == null || this.responseType.equals(SPARQLLanguageProcessor.QUERY_RESULT_TYPE_JSON)) { + assertEquals(2L, p.get(COMMON.RECEIVE_DATA_SIZE)); } } assertEquals(1, getWorker.getExecutedQueries()); @@ -156,10 +151,10 @@ private HttpWorker getWorker(String taskID) { } private HttpWorker getWorker(String taskID, Integer latencyFixed, Integer gaussianFixed) { - if(isPost){ - return new HttpPostWorker(taskID, getConnection(), this.queriesFile, "application/json", this.responseType, this.parameter, null, null, null, latencyFixed, gaussianFixed, 1); + if (this.isPost) { + return new HttpPostWorker(taskID, 1, getConnection(), this.queries, null, null, latencyFixed, gaussianFixed, this.parameter, this.responseType, "application/json"); } - return new HttpGetWorker(taskID, getConnection(), this.queriesFile, this.responseType, this.parameter, null, null, null, latencyFixed, gaussianFixed, 1); + return new HttpGetWorker(taskID, 1, getConnection(), this.queries, null, null, latencyFixed, gaussianFixed, this.parameter, this.responseType); } @@ -168,66 +163,60 @@ private Connection getConnection() { con.setName("test"); con.setPassword("test"); con.setUser("abc"); - con.setEndpoint(service); - con.setUpdateEndpoint(service); + con.setEndpoint(this.service); + con.setUpdateEndpoint(this.service); return con; } @Test public void testWait() throws InterruptedException { - String taskID="123/1/1/"; + String taskID = "123/1/1/"; HttpWorker getWorker = getWorker(taskID, this.fixedLatency, this.gaussianLatency); - InstancesQueryHandler qh = new InstancesQueryHandler(Lists.newArrayList(getWorker)); - qh.setOutputFolder(outputDir); - qh.generate(); ExecutorService executorService = Executors.newFixedThreadPool(1); executorService.submit(getWorker); - long waitMS=850; + long waitMS = 850; Thread.sleep(waitMS); getWorker.stopSending(); executorService.shutdownNow(); //get expected delay - int expectedDelay = 100+this.fixedLatency+this.gaussianLatency; - if(isPost){ - expectedDelay+=100; + int expectedDelay = 100 + this.fixedLatency + this.gaussianLatency; + if (this.isPost) { + expectedDelay += 100; } - double expectedQueries = waitMS*1.0/expectedDelay; - double deltaUp = waitMS*1.0/(expectedDelay+gaussianLatency); - double deltaDown = waitMS*1.0/(expectedDelay-gaussianLatency); - double delta = Math.ceil((deltaDown-deltaUp)/2); - assertEquals(expectedQueries, 1.0*getWorker.getExecutedQueries(), delta); + double expectedQueries = waitMS * 1.0 / expectedDelay; + double deltaUp = waitMS * 1.0 / (expectedDelay + this.gaussianLatency); + double deltaDown = waitMS * 1.0 / (expectedDelay - this.gaussianLatency); + double delta = Math.ceil((deltaDown - deltaUp) / 2); + assertEquals(expectedQueries, 1.0 * getWorker.getExecutedQueries(), delta); } @Test - public void testWorkflow() throws InterruptedException, IOException { + public void testWorkflow() throws InterruptedException { // check as long as not endsignal - String taskID="123/1/1/"; + String taskID = "123/1/1/"; int queryHash = FileUtils.getHashcodeFromFileContent(this.queriesFile); HttpWorker getWorker = getWorker(taskID); - InstancesQueryHandler qh = new InstancesQueryHandler(Lists.newArrayList(getWorker)); - qh.setOutputFolder(outputDir); - qh.generate(); ExecutorService executorService = Executors.newFixedThreadPool(1); executorService.submit(getWorker); Thread.sleep(450); getWorker.stopSending(); executorService.shutdownNow(); // check correct executedQueries - long expectedSize=4; - if(isPost){ - expectedSize=2; + long expectedSize = 4; + if (this.isPost) { + expectedSize = 2; } assertEquals(expectedSize, getWorker.getExecutedQueries()); // check pop query results Collection results = getWorker.popQueryResults(); - for(Properties p : results){ + for (Properties p : results) { assertEquals(queryHash, p.get(COMMON.QUERY_HASH)); } assertEquals(expectedSize, results.size()); - for(long i=1;i queriesFile; + private final int expectedExec; private String outputDir; - private int expectedExec; + + public UPDATEWorkerTest(String timerStrategy, Map queriesFile, int expectedExec) { + this.service = "http://localhost:8025/test"; + this.timerStrategy = timerStrategy; + this.queriesFile = queriesFile; + this.expectedExec = expectedExec; + //warmup + Map warmupQueries = new HashMap<>(); + warmupQueries.put("location", "src/test/resources/workers/single-query.txt"); + SPARQLWorker worker = new SPARQLWorker("", 1, getConnection(), warmupQueries, null, null, null, null, null, null); + worker.executeQuery("INSERT DATA {", "1"); + fastServerContainer.getTimes().clear(); + fastServerContainer.getEncodedAuth().clear(); + } @Parameterized.Parameters - public static Collection data(){ - Collection testData = new ArrayList(); - testData.add(new Object[]{"none", "src/test/resources/workers/updates", 4}); - testData.add(new Object[]{"fixed", "src/test/resources/workers/updates", 4}); - testData.add(new Object[]{"distributed", "src/test/resources/workers/updates", 4}); - testData.add(new Object[]{"none", "src/test/resources/workers/updates.txt", 3}); - testData.add(new Object[]{"fixed", "src/test/resources/workers/updates.txt", 3}); - testData.add(new Object[]{"distributed", "src/test/resources/workers/updates.txt", 3}); + public static Collection data() { + Collection testData = new ArrayList<>(); + + Map queries0 = new HashMap<>(); + queries0.put("location", "src/test/resources/workers/updates"); + queries0.put("format", "folder"); + testData.add(new Object[]{"none", queries0, 4}); + + Map queries1 = new HashMap<>(); + queries1.put("location", "src/test/resources/workers/updates"); + queries1.put("format", "folder"); + testData.add(new Object[]{"fixed", queries1, 4}); + + Map queries2 = new HashMap<>(); + queries2.put("location", "src/test/resources/workers/updates"); + queries2.put("format", "folder"); + testData.add(new Object[]{"distributed", queries2, 4}); + + Map queries3 = new HashMap<>(); + queries3.put("location", "src/test/resources/workers/updates.txt"); + testData.add(new Object[]{"none", queries3, 3}); + + Map queries4 = new HashMap<>(); + queries4.put("location", "src/test/resources/workers/updates.txt"); + testData.add(new Object[]{"fixed", queries4, 3}); + + Map queries5 = new HashMap<>(); + queries5.put("location", "src/test/resources/workers/updates.txt"); + testData.add(new Object[]{"distributed", queries5, 3}); return testData; } @@ -65,26 +98,14 @@ public static void stopServer() throws IOException { fastServer.stop(); } - public UPDATEWorkerTest(String timerStrategy, String queriesFile, int expectedExec){ - this.service="http://localhost:8025/test"; - this.timerStrategy=timerStrategy; - this.queriesFile=queriesFile; - this.expectedExec=expectedExec; - //warmup - SPARQLWorker worker = new SPARQLWorker("", getConnection(), this.queriesFile, null, null, null, null, null, null, 1); - worker.executeQuery("INSERT DATA {", "1"); - fastServerContainer.getTimes().clear(); - fastServerContainer.getEncodedAuth().clear(); - } - @Before - public void createDir(){ - this.outputDir= UUID.randomUUID().toString(); + public void createDir() { + this.outputDir = UUID.randomUUID().toString(); } @After public void cleanup() throws IOException { - FileUtils.deleteDirectory(new File(outputDir)); + FileUtils.deleteDirectory(new File(this.outputDir)); fastServerContainer.getTimes().clear(); fastServerContainer.getEncodedAuth().clear(); } @@ -95,13 +116,10 @@ public void cleanup() throws IOException { // correct waiting in sum @Test public void testWorkflow() throws InterruptedException { - String taskID="124/1/1"; - Integer timeLimit=2000; + String taskID = "124/1/1"; + int timeLimit = 2000; Connection con = getConnection(); - UPDATEWorker worker = new UPDATEWorker(taskID, con, this.queriesFile, this.timerStrategy, null, timeLimit, null, null, 1); - InstancesQueryHandler qh = new InstancesQueryHandler(Lists.newArrayList(worker)); - qh.setOutputFolder(this.outputDir); - qh.generate(); + UPDATEWorker worker = new UPDATEWorker(taskID, 1, con, this.queriesFile, timeLimit, null, null, null, this.timerStrategy); worker.run(); Instant now = worker.startTime; @@ -110,33 +128,33 @@ public void testWorkflow() throws InterruptedException { Set creds = fastServerContainer.getEncodedAuth(); assertEquals(1, creds.size()); - assertEquals(con.getUser()+":"+con.getPassword(), creds.iterator().next()); + assertEquals(con.getUser() + ":" + con.getPassword(), creds.iterator().next()); List requestTimes = fastServerContainer.getTimes(); long noOfQueries = worker.getNoOfQueries(); - Double fixedValue = timeLimit/noOfQueries*1.0; + Double fixedValue = timeLimit / noOfQueries * 1.0; Instant pastInstant = requestTimes.get(0); - long remainingQueries = noOfQueries-1; - long remainingTime=timeLimit-Double.valueOf(TimeUtils.durationInMilliseconds(now, pastInstant)).longValue(); - for(int i=1;i getDefaultQueryConfig() { + Map queries = new HashMap<>(); + queries.put("location", "src/test/resources/workers/single-query.txt"); + return queries; + } + @Test public void buildRequest() throws IOException { String query = "DELETE DATA { \"äöüÄÖÜß\" . }"; - HttpPostWorker postWorker = new HttpPostWorker(null, getConnection(), null, "application/sparql", null, null, null, null, null, null, null, 0); + HttpPostWorker postWorker = new HttpPostWorker(null, 0, getConnection(), getDefaultQueryConfig(), null, null, null, null, null, null, "application/sparql"); postWorker.buildRequest(query, null); HttpPost request = ((HttpPost) postWorker.request); From 0d17a6949a6f615266589c3708c4d9e2f3c031d1 Mon Sep 17 00:00:00 2001 From: frensing Date: Mon, 31 Oct 2022 19:24:26 +0200 Subject: [PATCH 16/47] refactor cliworkers to use new query handling --- .../cc/worker/impl/CLIInputFileWorker.java | 58 ++-- .../cc/worker/impl/CLIInputPrefixWorker.java | 29 +- .../iguana/cc/worker/impl/CLIInputWorker.java | 207 ++++++------ .../aksw/iguana/cc/worker/impl/CLIWorker.java | 151 ++++----- .../worker/impl/MultipleCLIInputWorker.java | 311 +++++++++--------- .../cc/worker/impl/CLIWorkersTests.java | 67 ++-- 6 files changed, 395 insertions(+), 428 deletions(-) diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/impl/CLIInputFileWorker.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/impl/CLIInputFileWorker.java index c3ecb6343..ca4335948 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/impl/CLIInputFileWorker.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/impl/CLIInputFileWorker.java @@ -7,46 +7,44 @@ import java.io.File; import java.io.IOException; import java.io.PrintWriter; +import java.util.Map; /** * Worker to execute a query against a CLI process, the connection.service will be the command to execute the query against. - * + *

* Assumes that the CLI process won't stop but will just accepts queries one after another and returns the results in the CLI output. * also assumes that the query has to be read from a file instead of plain input - * + *

* This worker can be set to be created multiple times in the background if one process will throw an error, a backup process was already created and can be used. * This is handy if the process won't just prints an error message, but simply exits. - * */ @Shorthand("CLIInputFileWorker") public class CLIInputFileWorker extends MultipleCLIInputWorker { + private final String dir; + public CLIInputFileWorker(String taskID, Integer workerID, Connection connection, Map queries, @Nullable Integer timeLimit, @Nullable Integer timeOut, @Nullable Integer fixedLatency, @Nullable Integer gaussianLatency, String initFinished, String queryFinished, String queryError, @Nullable Integer numberOfProcesses, String directory) { + super(taskID, workerID, connection, queries, timeLimit, timeOut, fixedLatency, gaussianLatency, initFinished, queryFinished, queryError, numberOfProcesses); + this.dir = directory; + } + + @Override + protected String writableQuery(String query) { + File f; + + try { + new File(this.dir).mkdirs(); + f = new File(this.dir + File.separator + "tmpquery.sparql"); + f.createNewFile(); + f.deleteOnExit(); + try (PrintWriter pw = new PrintWriter(f)) { + pw.print(query); + } + return f.getName(); + } catch (IOException e) { + e.printStackTrace(); + } + + return query; + } - private String dir; - - public CLIInputFileWorker(String taskID, Connection connection, String queriesFile, String initFinished, String queryFinished, String queryError, @Nullable Integer numberOfProcesses, String directory, @Nullable Integer timeOut, @Nullable Integer timeLimit, @Nullable Integer fixedLatency, @Nullable Integer gaussianLatency, Integer workerID) { - super(taskID, connection, queriesFile, initFinished, queryFinished, queryError, numberOfProcesses, timeOut, timeLimit, fixedLatency, gaussianLatency, workerID); - this.dir = directory; - } - - @Override - protected String writableQuery(String query) { - File f; - - try { - new File(dir).mkdirs(); - f = new File(dir+File.separator+"tmpquery.sparql"); - f.createNewFile(); - f.deleteOnExit(); - try(PrintWriter pw = new PrintWriter(f)){ - pw.print(query); - } - return f.getName(); - } catch (IOException e) { - e.printStackTrace(); - } - - return query; - } - } diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/impl/CLIInputPrefixWorker.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/impl/CLIInputPrefixWorker.java index edd79f309..281639236 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/impl/CLIInputPrefixWorker.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/impl/CLIInputPrefixWorker.java @@ -5,32 +5,33 @@ import org.aksw.iguana.commons.annotation.Nullable; import org.aksw.iguana.commons.annotation.Shorthand; +import java.util.Map; + /** * Worker to execute a query against a CLI process, the connection.service will be the command to execute the query against. - * + *

* Assumes that the CLI process won't stop but will just accepts queries one after another and returns the results in the CLI output. * also assumes that the query has to be prefixed and suffixed. * For example: SPARQL SELECT * {?s ?p ?o} ; whereas 'SPARQL' is the prefix and ';' is the suffix. - * + *

* This worker can be set to be created multiple times in the background if one process will throw an error, a backup process was already created and can be used. * This is handy if the process won't just prints an error message, but simply exits. - * */ @Shorthand("CLIInputPrefixWorker") public class CLIInputPrefixWorker extends MultipleCLIInputWorker { - private String prefix; - private String suffix; + private final String prefix; + private final String suffix; - public CLIInputPrefixWorker(String taskID, Connection connection, String queriesFile, String initFinished, String queryFinished, String queryError, @Nullable Integer numberOfProcesses, String queryPrefix, String querySuffix, @Nullable Integer timeOut, @Nullable Integer timeLimit, @Nullable Integer fixedLatency, @Nullable Integer gaussianLatency, Integer workerID) { - super(taskID, connection, queriesFile, initFinished, queryFinished, queryError, numberOfProcesses, timeOut, timeLimit, fixedLatency, gaussianLatency, workerID); - this.prefix = queryPrefix; - this.suffix = querySuffix; - } + public CLIInputPrefixWorker(String taskID, Integer workerID, Connection connection, Map queries, @Nullable Integer timeLimit, @Nullable Integer timeOut, @Nullable Integer fixedLatency, @Nullable Integer gaussianLatency, String initFinished, String queryFinished, String queryError, @Nullable Integer numberOfProcesses, String queryPrefix, String querySuffix) { + super(taskID, workerID, connection, queries, timeLimit, timeOut, fixedLatency, gaussianLatency, initFinished, queryFinished, queryError, numberOfProcesses); + this.prefix = queryPrefix; + this.suffix = querySuffix; + } - @Override - protected String writableQuery(String query) { - return prefix+" "+query+" "+suffix; - } + @Override + protected String writableQuery(String query) { + return this.prefix + " " + query + " " + this.suffix; + } } diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/impl/CLIInputWorker.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/impl/CLIInputWorker.java index 0f35a8d8c..aa067cc35 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/impl/CLIInputWorker.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/impl/CLIInputWorker.java @@ -3,7 +3,7 @@ import org.aksw.iguana.cc.config.elements.Connection; import org.aksw.iguana.cc.model.QueryExecutionStats; import org.aksw.iguana.cc.utils.CLIProcessManager; -import org.aksw.iguana.cc.worker.AbstractRandomQueryChooserWorker; +import org.aksw.iguana.cc.worker.AbstractWorker; import org.aksw.iguana.commons.annotation.Nullable; import org.aksw.iguana.commons.annotation.Shorthand; import org.aksw.iguana.commons.constants.COMMON; @@ -12,7 +12,7 @@ import java.io.IOException; import java.time.Instant; -import java.util.Random; +import java.util.Map; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; @@ -23,115 +23,102 @@ /** * Worker to execute a query against a CLI process, the connection.service will be the command to execute the query against. - * + *

* Assumes that the CLI process won't stop but will just accepts queries one after another and returns the results in the CLI output. - * */ @Shorthand("CLIInputWorker") -public class CLIInputWorker extends AbstractRandomQueryChooserWorker { - - private final Logger LOGGER = LoggerFactory.getLogger(getClass()); - - private Random queryChooser; - private Process process; - private String initFinished; - private String queryFinished; - private String error; - - public CLIInputWorker(String taskID, Connection connection, String queriesFile, String initFinished, String queryFinished, String queryError, @Nullable Integer timeOut, @Nullable Integer timeLimit, @Nullable Integer fixedLatency, @Nullable Integer gaussianLatency, Integer workerID) { - super(taskID, connection, queriesFile, timeOut, timeLimit, fixedLatency, gaussianLatency, workerID); - queryChooser = new Random(this.workerID); - this.initFinished = initFinished; - this.queryFinished = queryFinished; - this.error = queryError; - this.setWorkerProperties(); - - } - - private void setWorkerProperties() - { - queryChooser = new Random(this.workerID); - - // Create a CLI process, initialize it - this.process = CLIProcessManager.createProcess(this.con.getEndpoint()); - try { - CLIProcessManager.countLinesUntilStringOccurs(process, this.initFinished, this.error); //Init - } catch (IOException e) { - LOGGER.error("Exception while trying to wait for init of CLI Process",e); - } - } - - @Override - public void executeQuery(String query, String queryID) { - Instant start = Instant.now(); - - try { - // Create background thread that will watch the output of the process and prepare results - AtomicLong size = new AtomicLong(-1); - AtomicBoolean failed = new AtomicBoolean(false); - ExecutorService executor = Executors.newSingleThreadExecutor(); - executor.execute(new Runnable() { - - @Override - public void run() { - try { - LOGGER.debug("Process Alive: {}", process.isAlive()); - LOGGER.debug("Reader ready: {}", CLIProcessManager.isReaderReady(process)); - size.set(CLIProcessManager.countLinesUntilStringOccurs(process, queryFinished, error)); - } catch (IOException e) { - failed.set(true); - } - } - }); - - // Execute the query on the process - try { - if (process.isAlive()) { - CLIProcessManager.executeCommand(process, writableQuery(query)); - } else if (this.endSignal) { - super.addResults(new QueryExecutionStats (queryID, COMMON.QUERY_UNKNOWN_EXCEPTION, durationInMilliseconds(start, Instant.now()) )); - return; - } else { - super.addResults(new QueryExecutionStats (queryID, COMMON.QUERY_UNKNOWN_EXCEPTION, durationInMilliseconds(start, Instant.now()) )); - return; - } - } finally { - executor.shutdown(); - executor.awaitTermination((long)(double)this.timeOut, TimeUnit.MILLISECONDS); - } - - // At this point, query is executed and background thread has processed the results. - // Next, calculate time for benchmark. - double duration = durationInMilliseconds(start, Instant.now()); - - if (duration >= timeOut) { - super.addResults(new QueryExecutionStats (queryID, COMMON.QUERY_SOCKET_TIMEOUT, duration )); - return; - } else if (failed.get()) { - super.addResults(new QueryExecutionStats (queryID, COMMON.QUERY_UNKNOWN_EXCEPTION, duration )); - return; - } - - // SUCCESS - LOGGER.debug("Query successfully executed size: {}", size.get()); - super.addResults(new QueryExecutionStats (queryID, COMMON.QUERY_SUCCESS, duration, size.get() )); - } catch (IOException | InterruptedException e) { - LOGGER.warn("Exception while executing query ",e); - // ERROR - super.addResults(new QueryExecutionStats (queryID, COMMON.QUERY_UNKNOWN_EXCEPTION, durationInMilliseconds(start, Instant.now()) )); - } - } - - - protected String writableQuery(String query) { - return query; - } - - - - @Override - public void stopSending() { - super.stopSending(); - process.destroyForcibly(); - } +public class CLIInputWorker extends AbstractWorker { + + private final Logger LOGGER = LoggerFactory.getLogger(getClass()); + private final String initFinished; + private final String queryFinished; + private final String error; + private Process process; + + public CLIInputWorker(String taskID, Integer workerID, Connection connection, Map queries, @Nullable Integer timeLimit, @Nullable Integer timeOut, @Nullable Integer fixedLatency, @Nullable Integer gaussianLatency, String initFinished, String queryFinished, String queryError) { + super(taskID, workerID, connection, queries, timeLimit, timeOut, fixedLatency, gaussianLatency); + this.initFinished = initFinished; + this.queryFinished = queryFinished; + this.error = queryError; + this.setWorkerProperties(); + } + + private void setWorkerProperties() { + // Create a CLI process, initialize it + this.process = CLIProcessManager.createProcess(this.con.getEndpoint()); + try { + CLIProcessManager.countLinesUntilStringOccurs(process, this.initFinished, this.error); //Init + } catch (IOException e) { + LOGGER.error("Exception while trying to wait for init of CLI Process", e); + } + } + + @Override + public void executeQuery(String query, String queryID) { + Instant start = Instant.now(); + + try { + // Create background thread that will watch the output of the process and prepare results + AtomicLong size = new AtomicLong(-1); + AtomicBoolean failed = new AtomicBoolean(false); + ExecutorService executor = Executors.newSingleThreadExecutor(); + executor.execute(() -> { + try { + LOGGER.debug("Process Alive: {}", this.process.isAlive()); + LOGGER.debug("Reader ready: {}", CLIProcessManager.isReaderReady(this.process)); + size.set(CLIProcessManager.countLinesUntilStringOccurs(this.process, this.queryFinished, this.error)); + } catch (IOException e) { + failed.set(true); + } + }); + + // Execute the query on the process + try { + if (this.process.isAlive()) { + CLIProcessManager.executeCommand(this.process, writableQuery(query)); + } else if (this.endSignal) { + super.addResults(new QueryExecutionStats(queryID, COMMON.QUERY_UNKNOWN_EXCEPTION, durationInMilliseconds(start, Instant.now()))); + return; + } else { + super.addResults(new QueryExecutionStats(queryID, COMMON.QUERY_UNKNOWN_EXCEPTION, durationInMilliseconds(start, Instant.now()))); + return; + } + } finally { + executor.shutdown(); + executor.awaitTermination((long) (double) this.timeOut, TimeUnit.MILLISECONDS); + } + + // At this point, query is executed and background thread has processed the results. + // Next, calculate time for benchmark. + double duration = durationInMilliseconds(start, Instant.now()); + + if (duration >= this.timeOut) { + super.addResults(new QueryExecutionStats(queryID, COMMON.QUERY_SOCKET_TIMEOUT, duration)); + return; + } else if (failed.get()) { + super.addResults(new QueryExecutionStats(queryID, COMMON.QUERY_UNKNOWN_EXCEPTION, duration)); + return; + } + + // SUCCESS + LOGGER.debug("Query successfully executed size: {}", size.get()); + super.addResults(new QueryExecutionStats(queryID, COMMON.QUERY_SUCCESS, duration, size.get())); + } catch (IOException | InterruptedException e) { + LOGGER.warn("Exception while executing query ", e); + // ERROR + super.addResults(new QueryExecutionStats(queryID, COMMON.QUERY_UNKNOWN_EXCEPTION, durationInMilliseconds(start, Instant.now()))); + } + } + + + protected String writableQuery(String query) { + return query; + } + + + @Override + public void stopSending() { + super.stopSending(); + this.process.destroyForcibly(); + } } diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/impl/CLIWorker.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/impl/CLIWorker.java index 9f5bf3f6e..b7ebc7506 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/impl/CLIWorker.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/impl/CLIWorker.java @@ -2,7 +2,7 @@ import org.aksw.iguana.cc.config.elements.Connection; import org.aksw.iguana.cc.model.QueryExecutionStats; -import org.aksw.iguana.cc.worker.AbstractRandomQueryChooserWorker; +import org.aksw.iguana.cc.worker.AbstractWorker; import org.aksw.iguana.commons.annotation.Nullable; import org.aksw.iguana.commons.annotation.Shorthand; import org.aksw.iguana.commons.constants.COMMON; @@ -14,96 +14,85 @@ import java.net.URLEncoder; import java.nio.charset.StandardCharsets; import java.time.Instant; +import java.util.Map; import static org.aksw.iguana.commons.time.TimeUtils.durationInMilliseconds; /** * Worker to execute a query again a CLI process, the connection.service will be the command to execute the query against. - * + *

* command may look like the following

* cliprocess.sh $QUERY$ $USER$ $PASSWORD$ *
* whereas $QUERY$ will be exchanged with the actual query as well as user and password. * Further on it is possible to encode the query using $ENCODEDQUERY$ instead of $QUERY$ - * */ @Shorthand("CLIWorker") -public class CLIWorker extends AbstractRandomQueryChooserWorker { - - private final Logger LOGGER = LoggerFactory.getLogger(getClass()); - - - public CLIWorker(String taskID, Connection connection, String queriesFile, @Nullable Integer timeOut, @Nullable Integer timeLimit, @Nullable Integer fixedLatency, @Nullable Integer gaussianLatency, Integer workerID) { - super(taskID, connection, queriesFile, timeOut, timeLimit, fixedLatency, gaussianLatency, workerID); - } - - - @Override - public void executeQuery(String query, String queryID) { - Instant start = Instant.now(); - // use cli as service - String encodedQuery = URLEncoder.encode(query, StandardCharsets.UTF_8); - String queryCLI = getReplacedQuery(query, encodedQuery); - // execute queryCLI and read response - ProcessBuilder processBuilder = new ProcessBuilder().redirectErrorStream(true); - processBuilder.command(new String[] { "bash", "-c", queryCLI }); - try { - - Process process = processBuilder.start(); - - StringBuilder output = new StringBuilder(); - long size = -1; - - try (BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream()))) { - - String line; - // -1 as the first line should be the header - while ((line = reader.readLine()) != null) { - - output.append(line + "\n"); - size++; - } - } catch (Exception e) { - e.printStackTrace(); - } - int exitVal = process.waitFor(); - if (exitVal == 0) { - LOGGER.debug("Query successfully executed size: {}", size); - } else { - LOGGER.warn("Exit Value of Process was not 0, was {} ", exitVal); - super.addResults(new QueryExecutionStats(queryID, COMMON.QUERY_UNKNOWN_EXCEPTION, durationInMilliseconds(start, Instant.now()) )); - return; - } - super.addResults(new QueryExecutionStats(queryID, COMMON.QUERY_SUCCESS, durationInMilliseconds(start, Instant.now()), size )); - return; - } catch (Exception e) { - LOGGER.warn("Unknown Exception while executing query", e); - } - // ERROR - super.addResults(new QueryExecutionStats(queryID, COMMON.QUERY_UNKNOWN_EXCEPTION, durationInMilliseconds(start, Instant.now()) )); - } - - private String getReplacedQuery(String query, String encodedQuery) { - String queryCLI = this.con.getEndpoint().replace("$QUERY$", query); - queryCLI = queryCLI.replace("$ENCODEDQUERY$", encodedQuery); - - if (this.con.getUser() != null) { - queryCLI = queryCLI.replace("$USER$", this.con.getUser()); - } - else{ - queryCLI = queryCLI.replace("$USER$", ""); - - } - if (this.con.getPassword() != null) { - queryCLI = queryCLI.replace("$PASSWORD$", this.con.getPassword()); - } - else{ - queryCLI = queryCLI.replace("$PASSWORD$", ""); - - } - return queryCLI; - - } - - +public class CLIWorker extends AbstractWorker { + + private final Logger LOGGER = LoggerFactory.getLogger(getClass()); + + public CLIWorker(String taskID, Integer workerID, Connection connection, Map queries, @Nullable Integer timeLimit, @Nullable Integer timeOut, @Nullable Integer fixedLatency, @Nullable Integer gaussianLatency) { + super(taskID, workerID, connection, queries, timeLimit, timeOut, fixedLatency, gaussianLatency); + } + + @Override + public void executeQuery(String query, String queryID) { + Instant start = Instant.now(); + // use cli as service + String encodedQuery = URLEncoder.encode(query, StandardCharsets.UTF_8); + String queryCLI = getReplacedQuery(query, encodedQuery); + // execute queryCLI and read response + ProcessBuilder processBuilder = new ProcessBuilder().redirectErrorStream(true); + processBuilder.command("bash", "-c", queryCLI); + try { + + Process process = processBuilder.start(); + + long size = -1; + + try (BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream()))) { + + // -1 as the first line should be the header + while (reader.readLine() != null) { + size++; + } + } catch (Exception e) { + e.printStackTrace(); + } + int exitVal = process.waitFor(); + if (exitVal == 0) { + LOGGER.debug("Query successfully executed size: {}", size); + } else { + LOGGER.warn("Exit Value of Process was not 0, was {} ", exitVal); + super.addResults(new QueryExecutionStats(queryID, COMMON.QUERY_UNKNOWN_EXCEPTION, durationInMilliseconds(start, Instant.now()))); + return; + } + super.addResults(new QueryExecutionStats(queryID, COMMON.QUERY_SUCCESS, durationInMilliseconds(start, Instant.now()), size)); + return; + } catch (Exception e) { + LOGGER.warn("Unknown Exception while executing query", e); + } + // ERROR + super.addResults(new QueryExecutionStats(queryID, COMMON.QUERY_UNKNOWN_EXCEPTION, durationInMilliseconds(start, Instant.now()))); + } + + private String getReplacedQuery(String query, String encodedQuery) { + String queryCLI = this.con.getEndpoint().replace("$QUERY$", query); + queryCLI = queryCLI.replace("$ENCODEDQUERY$", encodedQuery); + + if (this.con.getUser() != null) { + queryCLI = queryCLI.replace("$USER$", this.con.getUser()); + } else { + queryCLI = queryCLI.replace("$USER$", ""); + + } + if (this.con.getPassword() != null) { + queryCLI = queryCLI.replace("$PASSWORD$", this.con.getPassword()); + } else { + queryCLI = queryCLI.replace("$PASSWORD$", ""); + + } + return queryCLI; + } } diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/impl/MultipleCLIInputWorker.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/impl/MultipleCLIInputWorker.java index f823455e0..8063276ff 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/impl/MultipleCLIInputWorker.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/impl/MultipleCLIInputWorker.java @@ -3,7 +3,7 @@ import org.aksw.iguana.cc.config.elements.Connection; import org.aksw.iguana.cc.model.QueryExecutionStats; import org.aksw.iguana.cc.utils.CLIProcessManager; -import org.aksw.iguana.cc.worker.AbstractRandomQueryChooserWorker; +import org.aksw.iguana.cc.worker.AbstractWorker; import org.aksw.iguana.commons.annotation.Nullable; import org.aksw.iguana.commons.annotation.Shorthand; import org.aksw.iguana.commons.constants.COMMON; @@ -13,7 +13,7 @@ import java.io.IOException; import java.time.Instant; import java.util.List; -import java.util.Random; +import java.util.Map; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; @@ -24,168 +24,157 @@ /** * Worker to execute a query against a CLI process, the connection.service will be the command to execute the query against. - * + *

* Assumes that the CLI process won't stop but will just accepts queries one after another and returns the results in the CLI output. - * + *

* This worker can be set to be created multiple times in the background if one process will throw an error, a backup process was already created and can be used. * This is handy if the process won't just prints an error message, but simply exits. - * */ @Shorthand("MultipleCLIInputWorker") -public class MultipleCLIInputWorker extends AbstractRandomQueryChooserWorker { - - private final Logger LOGGER = LoggerFactory.getLogger(getClass()); - - private Process currentProcess; - protected List processList; - protected int currentProcessId = 0; - private String initFinished; - private String queryFinished; - private String error; - protected int numberOfProcesses = 5; - - public MultipleCLIInputWorker(String taskID, Connection connection, String queriesFile, String initFinished, String queryFinished, String queryError, @Nullable Integer numberOfProcesses, @Nullable Integer timeOut, @Nullable Integer timeLimit, @Nullable Integer fixedLatency, @Nullable Integer gaussianLatency, Integer workerID) { - super(taskID, connection, queriesFile, timeOut, timeLimit, fixedLatency, gaussianLatency, workerID); - this.initFinished = initFinished; - this.queryFinished = queryFinished; - this.error = queryError; - if(numberOfProcesses!=null){ - this.numberOfProcesses=numberOfProcesses; - } - this.setWorkerProperties(); - - } - - private void setWorkerProperties() { - queryChooser = new Random(this.workerID); - // start cli input - - // Create processes, set first process as current process - this.processList = CLIProcessManager.createProcesses(this.numberOfProcesses, this.con.getEndpoint()); - this.currentProcess = processList.get(0); - - // Make sure that initialization is complete - for (Process value : processList) { - try { - CLIProcessManager.countLinesUntilStringOccurs(value, initFinished, error); - } catch (IOException e) { - LOGGER.error("Exception while trying to wait for init of CLI Process",e); - } - } - } - - - @Override - public void executeQuery(String query, String queryID) { - Instant start = Instant.now(); - // execute queryCLI and read response - try { - // Create background thread that will watch the output of the process and prepare results - AtomicLong size = new AtomicLong(-1); - AtomicBoolean failed = new AtomicBoolean(false); - ExecutorService executor = Executors.newSingleThreadExecutor(); - executor.execute(new Runnable() { - - @Override - public void run() { - try { - LOGGER.debug("Process Alive: {}", currentProcess.isAlive()); - LOGGER.debug("Reader ready: {}", CLIProcessManager.isReaderReady(currentProcess)); - size.set(CLIProcessManager.countLinesUntilStringOccurs(currentProcess, queryFinished, error)); - } catch (IOException e) { - failed.set(true); - } - } - }); - - // Execute the query on the process - try { - if (currentProcess.isAlive()) { - CLIProcessManager.executeCommand(currentProcess, writableQuery(query)); - } else if (this.endSignal) { - super.addResults(new QueryExecutionStats(queryID, COMMON.QUERY_UNKNOWN_EXCEPTION, durationInMilliseconds(start, Instant.now()) )); - return; - } else { - setNextProcess(); - super.addResults(new QueryExecutionStats(queryID, COMMON.QUERY_UNKNOWN_EXCEPTION, durationInMilliseconds(start, Instant.now()) )); - return; - } - } finally { - executor.shutdown(); - executor.awaitTermination((long) (double)this.timeOut, TimeUnit.MILLISECONDS); - } - - // At this point, query is executed and background thread has processed the results. - // Next, calculate time for benchmark. - double duration = durationInMilliseconds(start, Instant.now()); - - if (duration >= timeOut) { - setNextProcess(); - super.addResults(new QueryExecutionStats(queryID, COMMON.QUERY_SOCKET_TIMEOUT, duration )); - return; - } else if (failed.get()) { - if (!currentProcess.isAlive()) { - setNextProcess(); - } - super.addResults(new QueryExecutionStats(queryID, COMMON.QUERY_UNKNOWN_EXCEPTION, duration )); - return; - } - - // SUCCESS - LOGGER.debug("Query successfully executed size: {}", size.get()); - super.addResults(new QueryExecutionStats(queryID, COMMON.QUERY_SUCCESS, duration, size.get() )); - return; - } catch (IOException | InterruptedException e) { - LOGGER.warn("Exception while executing query ",e); - // ERROR - super.addResults(new QueryExecutionStats(queryID, COMMON.QUERY_UNKNOWN_EXCEPTION, durationInMilliseconds(start, Instant.now()) )); - } - } - - private void setNextProcess() { - int oldProcessId = currentProcessId; - currentProcessId = currentProcessId == processList.size() -1 ? 0 : currentProcessId + 1; - - // destroy old process - CLIProcessManager.destroyProcess(currentProcess); - if(oldProcessId== currentProcessId) { - try { - currentProcess.waitFor(); - } catch (InterruptedException e) { - LOGGER.error("Process was Interrupted",e); - } - } - - // create and initialize new process to replace previously destroyed process - Process replacementProcess = CLIProcessManager.createProcess(this.con.getEndpoint()); - try { - CLIProcessManager.countLinesUntilStringOccurs(replacementProcess, initFinished, error); // Init - processList.set(oldProcessId, replacementProcess); - } catch (IOException e) { - LOGGER.error("Process replacement didn't work", e); - } - - // finally, update current process - currentProcess = processList.get(currentProcessId); - } - - protected String writableQuery(String query) { - return query; - } - - - - - @Override - public void stopSending() { - super.stopSending(); - for (Process pr : processList) { - pr.destroyForcibly(); - try { - pr.waitFor(); - } catch (InterruptedException e) { - LOGGER.error("Process waitFor was Interrupted", e); - } - } - } +public class MultipleCLIInputWorker extends AbstractWorker { + + private final Logger LOGGER = LoggerFactory.getLogger(getClass()); + private final String initFinished; + private final String queryFinished; + private final String error; + protected List processList; + protected int currentProcessId = 0; + protected int numberOfProcesses = 5; + private Process currentProcess; + + public MultipleCLIInputWorker(String taskID, Integer workerID, Connection connection, Map queries, @Nullable Integer timeLimit, @Nullable Integer timeOut, @Nullable Integer fixedLatency, @Nullable Integer gaussianLatency, String initFinished, String queryFinished, String queryError, @Nullable Integer numberOfProcesses) { + super(taskID, workerID, connection, queries, timeLimit, timeOut, fixedLatency, gaussianLatency); + this.initFinished = initFinished; + this.queryFinished = queryFinished; + this.error = queryError; + if (numberOfProcesses != null) { + this.numberOfProcesses = numberOfProcesses; + } + this.setWorkerProperties(); + } + + private void setWorkerProperties() { + // start cli input + + // Create processes, set first process as current process + this.processList = CLIProcessManager.createProcesses(this.numberOfProcesses, this.con.getEndpoint()); + this.currentProcess = this.processList.get(0); + + // Make sure that initialization is complete + for (Process value : this.processList) { + try { + CLIProcessManager.countLinesUntilStringOccurs(value, initFinished, error); + } catch (IOException e) { + LOGGER.error("Exception while trying to wait for init of CLI Process", e); + } + } + } + + + @Override + public void executeQuery(String query, String queryID) { + Instant start = Instant.now(); + // execute queryCLI and read response + try { + // Create background thread that will watch the output of the process and prepare results + AtomicLong size = new AtomicLong(-1); + AtomicBoolean failed = new AtomicBoolean(false); + ExecutorService executor = Executors.newSingleThreadExecutor(); + executor.execute(() -> { + try { + LOGGER.debug("Process Alive: {}", this.currentProcess.isAlive()); + LOGGER.debug("Reader ready: {}", CLIProcessManager.isReaderReady(this.currentProcess)); + size.set(CLIProcessManager.countLinesUntilStringOccurs(this.currentProcess, this.queryFinished, this.error)); + } catch (IOException e) { + failed.set(true); + } + }); + + // Execute the query on the process + try { + if (this.currentProcess.isAlive()) { + CLIProcessManager.executeCommand(this.currentProcess, writableQuery(query)); + } else if (this.endSignal) { + super.addResults(new QueryExecutionStats(queryID, COMMON.QUERY_UNKNOWN_EXCEPTION, durationInMilliseconds(start, Instant.now()))); + return; + } else { + setNextProcess(); + super.addResults(new QueryExecutionStats(queryID, COMMON.QUERY_UNKNOWN_EXCEPTION, durationInMilliseconds(start, Instant.now()))); + return; + } + } finally { + executor.shutdown(); + executor.awaitTermination((long) (double) this.timeOut, TimeUnit.MILLISECONDS); + } + + // At this point, query is executed and background thread has processed the results. + // Next, calculate time for benchmark. + double duration = durationInMilliseconds(start, Instant.now()); + + if (duration >= this.timeOut) { + setNextProcess(); + super.addResults(new QueryExecutionStats(queryID, COMMON.QUERY_SOCKET_TIMEOUT, duration)); + return; + } else if (failed.get()) { + if (!this.currentProcess.isAlive()) { + setNextProcess(); + } + super.addResults(new QueryExecutionStats(queryID, COMMON.QUERY_UNKNOWN_EXCEPTION, duration)); + return; + } + + // SUCCESS + LOGGER.debug("Query successfully executed size: {}", size.get()); + super.addResults(new QueryExecutionStats(queryID, COMMON.QUERY_SUCCESS, duration, size.get())); + } catch (IOException | InterruptedException e) { + LOGGER.warn("Exception while executing query ", e); + // ERROR + super.addResults(new QueryExecutionStats(queryID, COMMON.QUERY_UNKNOWN_EXCEPTION, durationInMilliseconds(start, Instant.now()))); + } + } + + private void setNextProcess() { + int oldProcessId = this.currentProcessId; + this.currentProcessId = this.currentProcessId == this.processList.size() - 1 ? 0 : this.currentProcessId + 1; + + // destroy old process + CLIProcessManager.destroyProcess(this.currentProcess); + if (oldProcessId == this.currentProcessId) { + try { + this.currentProcess.waitFor(); + } catch (InterruptedException e) { + LOGGER.error("Process was Interrupted", e); + } + } + + // create and initialize new process to replace previously destroyed process + Process replacementProcess = CLIProcessManager.createProcess(this.con.getEndpoint()); + try { + CLIProcessManager.countLinesUntilStringOccurs(replacementProcess, this.initFinished, this.error); // Init + this.processList.set(oldProcessId, replacementProcess); + } catch (IOException e) { + LOGGER.error("Process replacement didn't work", e); + } + + // finally, update current process + this.currentProcess = this.processList.get(this.currentProcessId); + } + + protected String writableQuery(String query) { + return query; + } + + + @Override + public void stopSending() { + super.stopSending(); + for (Process pr : this.processList) { + pr.destroyForcibly(); + try { + pr.waitFor(); + } catch (InterruptedException e) { + LOGGER.error("Process waitFor was Interrupted", e); + } + } + } } diff --git a/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/worker/impl/CLIWorkersTests.java b/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/worker/impl/CLIWorkersTests.java index 81c02433d..8ca69f5e4 100644 --- a/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/worker/impl/CLIWorkersTests.java +++ b/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/worker/impl/CLIWorkersTests.java @@ -9,9 +9,7 @@ import java.io.File; import java.io.IOException; -import java.util.Collection; -import java.util.Properties; -import java.util.UUID; +import java.util.*; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -21,24 +19,23 @@ public class CLIWorkersTests { private File f; @Before - public void createFile(){ + public void createFile() { String file = UUID.randomUUID().toString(); this.f = new File(file); - ; } @After - public void deleteFile(){ - f.delete(); + public void deleteFile() { + this.f.delete(); } @Test - public void checkMultipleProcesses(){ + public void checkMultipleProcesses() { Connection con = new Connection(); - con.setEndpoint("src/test/resources/cli/echoinput.sh "+f.getAbsolutePath()); - MultipleCLIInputWorker worker = new MultipleCLIInputWorker("123/1/1", con, "src/test/resources/update/empty.nt", "init finished", "rows", "query fail", 2, null, null, null,null, 1); + con.setEndpoint("src/test/resources/cli/echoinput.sh " + f.getAbsolutePath()); + MultipleCLIInputWorker worker = new MultipleCLIInputWorker("123/1/1", 1, con, getQueryConfig(), null, null, null, null, "init finished", "rows", "query fail", 2); assertEquals(2, worker.processList.size()); - for(Process p : worker.processList){ + for (Process p : worker.processList) { assertTrue(p.isAlive()); } //should run normally @@ -50,7 +47,7 @@ public void checkMultipleProcesses(){ assertEquals(1, worker.currentProcessId); assertEquals(2, worker.processList.size()); - for(Process p : worker.processList){ + for (Process p : worker.processList) { assertTrue(p.isAlive()); } worker.executeQuery("quit", "2"); @@ -63,12 +60,12 @@ public void checkFileInput() throws IOException { //check if file is created and used Connection con = new Connection(); String dir = UUID.randomUUID().toString(); - con.setEndpoint("src/test/resources/cli/echoinput.sh "+f.getAbsolutePath()); - CLIInputFileWorker worker = new CLIInputFileWorker("123/1/1", con, "src/test/resources/update/empty.nt", "init finished", "rows", "query fail", 1, dir, null, null, null, null, 1); + con.setEndpoint("src/test/resources/cli/echoinput.sh " + f.getAbsolutePath()); + CLIInputFileWorker worker = new CLIInputFileWorker("123/1/1", 1, con, getQueryConfig(), null, null, null, null, "init finished", "rows", "query fail", 1, dir); worker.executeQuery("test", "1"); - assertEquals("test", FileUtils.readFile(dir+File.separator+"tmpquery.sparql")); + assertEquals("test", FileUtils.readFile(dir + File.separator + "tmpquery.sparql")); worker.executeQuery("SELECT whatever", "1"); - assertEquals("SELECT whatever", FileUtils.readFile(dir+File.separator+"tmpquery.sparql")); + assertEquals("SELECT whatever", FileUtils.readFile(dir + File.separator + "tmpquery.sparql")); assertEquals("tmpquery.sparql\ntmpquery.sparql\n", FileUtils.readFile(f.getAbsolutePath())); org.apache.commons.io.FileUtils.deleteDirectory(new File(dir)); @@ -81,8 +78,8 @@ public void checkInput() throws IOException { // check if connection stays Connection con = new Connection(); - con.setEndpoint("src/test/resources/cli/echoinput.sh "+f.getAbsolutePath()); - CLIInputWorker worker = new CLIInputWorker("123/1/1", con, "src/test/resources/update/empty.nt", "init finished", "rows", "query fail", null, null, null, null, 1); + con.setEndpoint("src/test/resources/cli/echoinput.sh " + f.getAbsolutePath()); + CLIInputWorker worker = new CLIInputWorker("123/1/1", 1, con, getQueryConfig(), null, null, null, null, "init finished", "rows", "query fail"); worker.executeQuery("test", "1"); worker.executeQuery("SELECT whatever", "1"); assertEquals("test\nSELECT whatever\n", FileUtils.readFile(f.getAbsolutePath())); @@ -90,10 +87,10 @@ public void checkInput() throws IOException { assertEquals(2, succeededResults.size()); Properties succ = succeededResults.iterator().next(); assertEquals(COMMON.QUERY_SUCCESS, succ.get(COMMON.RECEIVE_DATA_SUCCESS)); - assertEquals(3l, succ.get(COMMON.RECEIVE_DATA_SIZE)); + assertEquals(3L, succ.get(COMMON.RECEIVE_DATA_SIZE)); succ = succeededResults.iterator().next(); assertEquals(COMMON.QUERY_SUCCESS, succ.get(COMMON.RECEIVE_DATA_SUCCESS)); - assertEquals(3l, succ.get(COMMON.RECEIVE_DATA_SIZE)); + assertEquals(3L, succ.get(COMMON.RECEIVE_DATA_SIZE)); // check fail worker.executeQuery("fail", "2"); @@ -102,7 +99,7 @@ public void checkInput() throws IOException { assertEquals(1, failedResults.size()); Properties fail = failedResults.iterator().next(); assertEquals(COMMON.QUERY_UNKNOWN_EXCEPTION, fail.get(COMMON.RECEIVE_DATA_SUCCESS)); - assertEquals(0l, fail.get(COMMON.RECEIVE_DATA_SIZE)); + assertEquals(0L, fail.get(COMMON.RECEIVE_DATA_SIZE)); worker.stopSending(); @@ -113,8 +110,8 @@ public void checkPrefix() throws IOException { // check if connection stays Connection con = new Connection(); - con.setEndpoint("src/test/resources/cli/echoinput.sh "+f.getAbsolutePath()); - CLIInputPrefixWorker worker = new CLIInputPrefixWorker("123/1/1", con, "src/test/resources/update/empty.nt", "init finished", "rows", "query fail", 1, "prefix", "suffix", null, null, null, null, 1); + con.setEndpoint("src/test/resources/cli/echoinput.sh " + f.getAbsolutePath()); + CLIInputPrefixWorker worker = new CLIInputPrefixWorker("123/1/1", 1, con, getQueryConfig(), null, null, null, null, "init finished", "rows", "query fail", 1, "prefix", "suffix"); worker.executeQuery("test", "1"); worker.executeQuery("SELECT whatever", "1"); assertEquals("prefix test suffix\nprefix SELECT whatever suffix\n", FileUtils.readFile(f.getAbsolutePath())); @@ -122,10 +119,10 @@ public void checkPrefix() throws IOException { assertEquals(2, succeededResults.size()); Properties succ = succeededResults.iterator().next(); assertEquals(COMMON.QUERY_SUCCESS, succ.get(COMMON.RECEIVE_DATA_SUCCESS)); - assertEquals(3l, succ.get(COMMON.RECEIVE_DATA_SIZE)); + assertEquals(3L, succ.get(COMMON.RECEIVE_DATA_SIZE)); succ = succeededResults.iterator().next(); assertEquals(COMMON.QUERY_SUCCESS, succ.get(COMMON.RECEIVE_DATA_SUCCESS)); - assertEquals(3l, succ.get(COMMON.RECEIVE_DATA_SIZE)); + assertEquals(3L, succ.get(COMMON.RECEIVE_DATA_SIZE)); // check fail worker.executeQuery("fail", "2"); @@ -134,7 +131,7 @@ public void checkPrefix() throws IOException { assertEquals(1, failedResults.size()); Properties fail = failedResults.iterator().next(); assertEquals(COMMON.QUERY_UNKNOWN_EXCEPTION, fail.get(COMMON.RECEIVE_DATA_SUCCESS)); - assertEquals(0l, fail.get(COMMON.RECEIVE_DATA_SIZE)); + assertEquals(0L, fail.get(COMMON.RECEIVE_DATA_SIZE)); worker.stopSending(); } @@ -146,21 +143,27 @@ public void checkCLI() throws IOException { con.setUser("user1"); con.setPassword("pwd"); - con.setEndpoint("/bin/echo \"$QUERY$ $USER$:$PASSWORD$ $ENCODEDQUERY$\" > "+f.getAbsolutePath()); - CLIWorker worker = new CLIWorker("123/1/1", con, "src/test/resources/update/empty.nt", null, null, null, null, 1); + con.setEndpoint("/bin/echo \"$QUERY$ $USER$:$PASSWORD$ $ENCODEDQUERY$\" > " + f.getAbsolutePath()); + CLIWorker worker = new CLIWorker("123/1/1", 1, con, getQueryConfig(), null, null, null, null); worker.executeQuery("test ()", "1"); String content = FileUtils.readFile(f.getAbsolutePath()); assertEquals("test () user1:pwd test+%28%29\n", content); con = new Connection(); - con.setEndpoint("/bin/echo \"$QUERY$ $USER$:$PASSWORD$ $ENCODEDQUERY$\" > "+f.getAbsolutePath()+" | /bin/printf \"HeaderDoesNotCount\na\na\""); - worker = new CLIWorker("123/1/1", con, "src/test/resources/update/empty.nt", null, null, null, null, 1); + con.setEndpoint("/bin/echo \"$QUERY$ $USER$:$PASSWORD$ $ENCODEDQUERY$\" > " + f.getAbsolutePath() + " | /bin/printf \"HeaderDoesNotCount\na\na\""); + worker = new CLIWorker("123/1/1", 1, con, getQueryConfig(), null, null, null, null); worker.executeQuery("test ()", "1"); content = FileUtils.readFile(f.getAbsolutePath()); assertEquals("test () : test+%28%29\n", content); - Collection results = worker.popQueryResults(); + Collection results = worker.popQueryResults(); assertEquals(1, results.size()); Properties p = results.iterator().next(); - assertEquals(2l, p.get(COMMON.RECEIVE_DATA_SIZE)); + assertEquals(2L, p.get(COMMON.RECEIVE_DATA_SIZE)); + } + + private Map getQueryConfig() { + Map config = new HashMap<>(); + config.put("location", "src/test/resources/updates/empty.nt"); + return config; } } From f8d53b0634cdc4099a27e8b7448f8f0aeba7645e Mon Sep 17 00:00:00 2001 From: frensing Date: Mon, 31 Oct 2022 21:41:05 +0200 Subject: [PATCH 17/47] refactor stresstest --- .../aksw/iguana/cc/tasks/impl/Stresstest.java | 668 ++++++++---------- .../iguana/cc/tasks/impl/StresstestTest.java | 79 +-- .../aksw/iguana/cc/worker/MockupWorker.java | 40 +- 3 files changed, 364 insertions(+), 423 deletions(-) diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/tasks/impl/Stresstest.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/tasks/impl/Stresstest.java index 95a4763f8..f392cfee7 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/tasks/impl/Stresstest.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/tasks/impl/Stresstest.java @@ -1,13 +1,7 @@ -/** - * - */ package org.aksw.iguana.cc.tasks.impl; import org.aksw.iguana.cc.config.CONSTANTS; import org.aksw.iguana.cc.config.elements.Connection; -import org.aksw.iguana.cc.query.QueryHandler; -import org.aksw.iguana.cc.query.QueryHandlerFactory; -import org.aksw.iguana.cc.query.impl.InstancesQueryHandler; import org.aksw.iguana.cc.tasks.AbstractTask; import org.aksw.iguana.cc.worker.Worker; import org.aksw.iguana.cc.worker.WorkerFactory; @@ -19,7 +13,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.FileNotFoundException; import java.io.IOException; import java.io.StringWriter; import java.time.Instant; @@ -37,357 +30,314 @@ */ @Shorthand("Stresstest") public class Stresstest extends AbstractTask { - - - private static final Logger LOGGER = LoggerFactory - .getLogger(Stresstest.class); - - private ArrayList workerConfig; - private LinkedHashMap warmupConfig; - - private Double timeLimit; - private Long noOfQueryMixes; - protected List workers = new LinkedList(); - private Instant startTime; - private String qhClassName; - private Long noOfWorkers= 0L; - protected String qhCacheFolder = "queryInstances"; - - private Double warmupTimeMS; - - - - private HashMap qhConfig; - private List warmupWorkers = new ArrayList<>(); - private HashMap warmupQHConfig; - private String warmupQHClass; - - - public Stresstest(Integer timeLimit, ArrayList workers, LinkedHashMap queryHandler) throws FileNotFoundException { - this(timeLimit, workers, queryHandler, null); - } - - public Stresstest(Integer timeLimit, ArrayList workers, LinkedHashMap queryHandler, LinkedHashMap warmup) throws FileNotFoundException { - this.timeLimit=timeLimit.doubleValue(); - this.workerConfig = workers; - this.qhConfig = queryHandler; - this.warmupConfig = warmup; - } - - public Stresstest(ArrayList workers, LinkedHashMap queryHandler, Integer noOfQueryMixes) throws FileNotFoundException { - this(workers, queryHandler, null, noOfQueryMixes); - } - - public Stresstest(ArrayList workers, LinkedHashMap queryHandler, LinkedHashMap warmup, Integer noOfQueryMixes) throws FileNotFoundException { - this.noOfQueryMixes=noOfQueryMixes.longValue(); - this.workerConfig = workers; - this.qhConfig = queryHandler; - this.warmupConfig = warmup; - } - - private void setConfig(ArrayList workers, HashMap queryHandler, HashMap warmup){ - - noOfWorkers+=createWorkers(workers, this.workers, this.timeLimit); - //let TypedFactory create queryHandlerConfiguration from className and configuration and add Workers - this.qhClassName = queryHandler.get("className").toString(); - this.qhConfig = (HashMap)queryHandler.getOrDefault("configuration", new HashMap<>()); - qhConfig.put("workers", this.workers); - - //If warmup - if(warmup!=null){ - //set time - this.warmupTimeMS = ((Integer) warmup.get("timeLimit")).doubleValue(); - //set warmup workers - ArrayList warmupWorkerConfig = (ArrayList) warmup.get("workers"); - createWorkers(warmupWorkerConfig, this.warmupWorkers, this.warmupTimeMS); - //if warmup uses a different queryHandler than the actual one create the query handler - createWarmupQueryHandler(warmup); - } - addMetaData(); - } - - private void createWarmupQueryHandler(HashMap warmup) { - if(warmup.containsKey("queryHandler")){ - HashMap warmupQueryHandler = (HashMap) warmup.get("queryHandler"); - this.warmupQHClass = warmupQueryHandler.get("className").toString(); - this.warmupQHConfig = (HashMap)warmupQueryHandler.getOrDefault("configuration", new HashMap<>()); - this.warmupQHConfig.put("workers", this.warmupWorkers); - }else{ - //otherwise use default - this.warmupQHClass = qhClassName; - //create copy of the current configuration - this.warmupQHConfig = new HashMap(qhConfig); - this.warmupQHConfig.put("workers", this.warmupWorkers); - } - } - - private int createWorkers(ArrayList workers, List workersToAddTo, Double timeLimit){ - int noOfWorkers=0; - for(HashMap workerConfig : workers){ - noOfWorkers += createWorker(workerConfig, workersToAddTo, timeLimit, noOfWorkers); - } - return noOfWorkers; - } - - private int createWorker(HashMap workerConfig, List workersToAddTo, Double timeLimit, Integer baseID) { - //let TypedFactory create from className and configuration - String className = workerConfig.remove("className").toString(); - //if shorthand classname is used, exchange to full classname - Integer threads = (Integer)workerConfig.remove("threads"); - workerConfig.put("connection", con); - workerConfig.put("taskID", taskID); - if(timeLimit!=null) - workerConfig.put("timeLimit", timeLimit.intValue()); - for(int i=0;i props = worker.popQueryResults(); - if(props == null){ - return; - } - - for (Properties results : props) { - try { - - // send results via RabbitMQ - LOGGER.debug("[TaskID: {{}}] Send results", taskID); - this.sendResults(results); - LOGGER.debug("[TaskID: {{}}] results could be send", taskID); - } catch (IOException e) { - LOGGER.error("[TaskID: {{}}] Could not send results due to exc.",taskID, e); - LOGGER.error("[TaskID: {{}}] Results: {{}}", taskID, results); - } - } - } - - - @Override - public void close() { - super.close(); - - } - - protected long warmup() { - if(warmupTimeMS==null||warmupTimeMS==0l) { - return 0; - } - if(warmupWorkers.size()==0) { - return 0; - } - LOGGER.info("[TaskID: {{}}] will start {{}}ms warmup now using {} no of workers in total.", taskID, warmupTimeMS, warmupWorkers.size()); - return executeWarmup(warmupWorkers); - } - - - private long executeWarmup(List warmupWorkers) { - ExecutorService exec = Executors.newFixedThreadPool(2); - for(Worker worker : warmupWorkers) { - exec.submit(worker); - } - //wait as long as needed - Instant start = Instant.now(); - exec.shutdown(); - while(durationInMilliseconds(start, Instant.now()) <= warmupTimeMS) { - //clean up RAM - for(Worker worker: warmupWorkers) { - worker.popQueryResults(); - } - try { - TimeUnit.MILLISECONDS.sleep(50); - }catch(Exception e) { - LOGGER.error("Could not warmup "); - } - } - for(Worker worker : warmupWorkers) { - worker.stopSending(); - } - try { - exec.awaitTermination(5, TimeUnit.SECONDS); - - } catch (InterruptedException e) { - LOGGER.warn("[TaskID: {{}}] Warmup. Could not await Termination of Workers.", taskID); - } - try { - exec.shutdownNow(); - }catch(Exception e1) { - LOGGER.error("Shutdown problems ", e1); - } - //clear up - long queriesExec = 0; - for(Worker w : warmupWorkers){ - queriesExec+=w.getExecutedQueries(); - } - warmupWorkers.clear(); - LOGGER.info("[TaskID: {{}}] Warmup finished.", taskID); - return queriesExec; - } - - /** - * Checks if restriction (e.g. timelimit or noOfQueryMixes for each Worker) - * occurs - * - * @return true if restriction occurs, false otherwise - */ - protected boolean isFinished() { - if (timeLimit !=null) { - - Instant current = Instant.now(); - double passed_time = timeLimit - durationInMilliseconds(this.startTime, current); - return passed_time <= 0D; - } - else if (noOfQueryMixes != null) { - - // use noOfQueries of SPARQLWorkers (as soon as a worker hit the noOfQueries, it - // will stop sending results - // UpdateWorker are allowed to execute all their updates - boolean endFlag=true; - for (Worker worker : workers) { - long queriesInMix = 0; - - LOGGER.debug("No of query Mixes: {} , queriesInMix {}", worker.getExecutedQueries(),noOfQueryMixes); - //Check for each worker, if the - if (worker.hasExecutedNoOfQueryMixes(noOfQueryMixes)) { - if(!worker.isTerminated()) { - //if the worker was not already terminated, send last results, as tehy will not be sended afterwards - sendWorkerResult(worker); - } - worker.stopSending(); - } - else { - endFlag = false; - } - - } - return endFlag; - } - LOGGER.error("Neither time limit nor NoOfQueryMixes is set. executing task now"); - return true; - } - - public long getExecutedQueries(){ - long ret = 0; - for(Worker worker: workers){ - ret += worker.getExecutedQueries(); - } - return ret; - } + private static final Logger LOGGER = LoggerFactory.getLogger(Stresstest.class); + + private final Map warmupConfig; + private final List warmupWorkers = new ArrayList<>(); + private final List> workerConfig; + protected List workers = new LinkedList<>(); + protected String cacheFolder = "queryInstances"; + private Double warmupTimeMS; + private Double timeLimit; + private Long noOfQueryMixes; + private Instant startTime; + + + public Stresstest(Integer timeLimit, List> workers) { + this(timeLimit, workers, null); + } + + public Stresstest(Integer timeLimit, List> workers, Map warmup) { + this.timeLimit = timeLimit.doubleValue(); + this.workerConfig = workers; + this.warmupConfig = warmup; + } + + public Stresstest(List> workers, Integer noOfQueryMixes) { + this(workers, null, noOfQueryMixes); + } + + public Stresstest(List> workers, Map warmup, Integer noOfQueryMixes) { + this.noOfQueryMixes = noOfQueryMixes.longValue(); + this.workerConfig = workers; + this.warmupConfig = warmup; + } + + private void initWorkers() { + if (this.warmupConfig != null) { + createWarmupWorkers(); + } + createWorkers(); + } + + private void createWarmupWorkers() { + this.warmupTimeMS = ((Integer) this.warmupConfig.get("timeLimit")).doubleValue(); + + List> warmupWorkerConfig = (List>) this.warmupConfig.get("workers"); + createWorkers(warmupWorkerConfig, this.warmupWorkers, this.warmupTimeMS); + } + + private void createWorkers() { + createWorkers(this.workerConfig, this.workers, this.timeLimit); + } + + private void createWorkers(List> workers, List workersToAddTo, Double timeLimit) { + int workerID = 0; + for (Map workerConfig : workers) { + workerID += createWorker(workerConfig, workersToAddTo, timeLimit, workerID); + } + } + + private int createWorker(Map workerConfig, List workersToAddTo, Double timeLimit, Integer baseID) { + //let TypedFactory create from className and configuration + String className = workerConfig.remove("className").toString(); + //if shorthand classname is used, exchange to full classname + workerConfig.put("connection", this.con); + workerConfig.put("taskID", this.taskID); + addOutputFolder(workerConfig); + + if (timeLimit != null) { + workerConfig.put("timeLimit", timeLimit.intValue()); + } + Integer threads = (Integer) workerConfig.remove("threads"); + for (int i = 0; i < threads; i++) { + workerConfig.put("workerID", baseID + i); + Worker worker = new WorkerFactory().create(className, workerConfig); + if (this.noOfQueryMixes != null) { + worker.endAtNoOfQueryMixes(this.noOfQueryMixes); + } + workersToAddTo.add(worker); + } + return threads; + } + + private void addOutputFolder(Map workerConfig) { + Map queries = (Map) workerConfig.getOrDefault("queries", new HashMap<>()); + queries.put("outputFolder", this.cacheFolder); + } + + public void generateTripleStats() { + // TODO How are these Triplestats used? + + StringWriter sw = new StringWriter(); + for (Worker worker : this.workers) { + Model tripleStats = worker.getQueryHandler().getTripleStats(this.taskID); + RDFDataMgr.write(sw, tripleStats, RDFFormat.NTRIPLES); + } + this.metaData.put(COMMON.SIMPLE_TRIPLE_KEY, sw.toString()); + //this.metaData.put(COMMON.QUERY_STATS, tripleStats); + } + + /** + * Add extra Meta Data + */ + @Override + public void addMetaData() { + super.addMetaData(); + Properties extraMeta = new Properties(); + if (this.timeLimit != null) + extraMeta.put(CONSTANTS.TIME_LIMIT, this.timeLimit); + if (this.noOfQueryMixes != null) + extraMeta.put(CONSTANTS.NO_OF_QUERY_MIXES, this.noOfQueryMixes); + extraMeta.put("noOfWorkers", this.workers.size()); + this.metaData.put(COMMON.EXTRA_META_KEY, extraMeta); + } + + + @Override + public void init(String[] ids, String dataset, Connection connection, String taskName) { + super.init(ids, dataset, connection, taskName); + + initWorkers(); + addMetaData(); + generateTripleStats(); + } + + /* + * (non-Javadoc) + * + * @see org.aksw.iguana.cc.tasks.Task#start() + */ + @Override + public void execute() { + warmup(); + LOGGER.info("Task with ID {{}} will be executed now", this.taskID); + // Execute each Worker in ThreadPool + ExecutorService executor = Executors.newFixedThreadPool(this.workers.size()); + this.startTime = Instant.now(); + for (Worker worker : this.workers) { + executor.execute(worker); + } + LOGGER.info("[TaskID: {{}}]All {{}} workers have been started", this.taskID, this.workers.size()); + // wait timeLimit or noOfQueries + executor.shutdown(); + while (!isFinished()) { + // check if worker has results yet + for (Worker worker : this.workers) { + // if so send all results buffered + sendWorkerResult(worker); + } + loopSleep(); + } + LOGGER.debug("Sending stop signal to workers"); + // tell all workers to stop sending properties, thus the await termination will + // be safe with the results + for (Worker worker : this.workers) { + worker.stopSending(); + } + // Wait 5seconds so the workers can stop themselves, otherwise they will be + // stopped + try { + LOGGER.debug("Will shutdown now..."); + + LOGGER.info("[TaskID: {{}}] Will shutdown and await termination in 5s.", this.taskID); + boolean finished = executor.awaitTermination(5, TimeUnit.SECONDS); + LOGGER.info("[TaskID: {{}}] Task completed. Thread finished status {}", this.taskID, finished); + } catch (InterruptedException e) { + LOGGER.error("[TaskID: {{}}] Could not shutdown Threads/Workers due to ...", this.taskID); + LOGGER.error("... Exception: ", e); + try { + executor.shutdownNow(); + } catch (Exception e1) { + LOGGER.error("Problems shutting down", e1); + } + } + } + + private void loopSleep() { + try { + TimeUnit.MILLISECONDS.sleep(100); + } catch (Exception e) { + //shouldn't be thrown except something else really went wrong + LOGGER.error("Loop sleep did not work.", e); + } + } + + private void sendWorkerResult(Worker worker) { + Collection props = worker.popQueryResults(); + if (props == null) { + return; + } + + for (Properties results : props) { + try { + + // send results via RabbitMQ + LOGGER.debug("[TaskID: {{}}] Send results", this.taskID); + this.sendResults(results); + LOGGER.debug("[TaskID: {{}}] results could be send", this.taskID); + } catch (IOException e) { + LOGGER.error("[TaskID: {{}}] Could not send results due to exc.", this.taskID, e); + LOGGER.error("[TaskID: {{}}] Results: {{}}", this.taskID, results); + } + } + } + + + @Override + public void close() { + super.close(); + } + + protected long warmup() { + if (this.warmupTimeMS == null || this.warmupTimeMS == 0L) { + return 0; + } + if (this.warmupWorkers.size() == 0) { + return 0; + } + LOGGER.info("[TaskID: {{}}] will start {{}}ms warmup now using {} no of workers in total.", this.taskID, this.warmupTimeMS, this.warmupWorkers.size()); + return executeWarmup(this.warmupWorkers); + } + + + private long executeWarmup(List warmupWorkers) { + ExecutorService exec = Executors.newFixedThreadPool(2); + for (Worker worker : warmupWorkers) { + exec.submit(worker); + } + //wait as long as needed + Instant start = Instant.now(); + exec.shutdown(); + while (durationInMilliseconds(start, Instant.now()) <= this.warmupTimeMS) { + //clean up RAM + for (Worker worker : warmupWorkers) { + worker.popQueryResults(); + } + try { + TimeUnit.MILLISECONDS.sleep(50); + } catch (Exception e) { + LOGGER.error("Could not warmup "); + } + } + for (Worker worker : warmupWorkers) { + worker.stopSending(); + } + try { + exec.awaitTermination(5, TimeUnit.SECONDS); + + } catch (InterruptedException e) { + LOGGER.warn("[TaskID: {{}}] Warmup. Could not await Termination of Workers.", this.taskID); + } + try { + exec.shutdownNow(); + } catch (Exception e1) { + LOGGER.error("Shutdown problems ", e1); + } + //clear up + long queriesExec = 0; + for (Worker w : warmupWorkers) { + queriesExec += w.getExecutedQueries(); + } + warmupWorkers.clear(); + LOGGER.info("[TaskID: {{}}] Warmup finished.", this.taskID); + return queriesExec; + } + + /** + * Checks if restriction (e.g. timelimit or noOfQueryMixes for each Worker) + * occurs + * + * @return true if restriction occurs, false otherwise + */ + protected boolean isFinished() { + if (this.timeLimit != null) { + + Instant current = Instant.now(); + double passed_time = this.timeLimit - durationInMilliseconds(this.startTime, current); + return passed_time <= 0D; + } else if (this.noOfQueryMixes != null) { + + // use noOfQueries of SPARQLWorkers (as soon as a worker hit the noOfQueries, it + // will stop sending results + // UpdateWorker are allowed to execute all their updates + boolean endFlag = true; + for (Worker worker : this.workers) { + LOGGER.debug("No of query Mixes: {} , queriesInMix {}", worker.getExecutedQueries(), this.noOfQueryMixes); + //Check for each worker, if the + if (worker.hasExecutedNoOfQueryMixes(this.noOfQueryMixes)) { + if (!worker.isTerminated()) { + //if the worker was not already terminated, send last results, as tehy will not be sended afterwards + sendWorkerResult(worker); + } + worker.stopSending(); + } else { + endFlag = false; + } + + } + return endFlag; + } + LOGGER.error("Neither time limit nor NoOfQueryMixes is set. executing task now"); + return true; + } + + public long getExecutedQueries() { + long ret = 0; + for (Worker worker : this.workers) { + ret += worker.getExecutedQueries(); + } + return ret; + } } diff --git a/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/tasks/impl/StresstestTest.java b/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/tasks/impl/StresstestTest.java index 66d60e2e4..35c02d6fd 100644 --- a/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/tasks/impl/StresstestTest.java +++ b/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/tasks/impl/StresstestTest.java @@ -22,32 +22,26 @@ public class StresstestTest { // test correct # of worker creation, meta data and warmup - private String[] queries = new String[]{"a", "b"}; - private String[] queries2 = new String[]{"b", "c"}; + private final String[] queries = new String[]{"a", "b"}; + private final String[] queries2 = new String[]{"b", "c"}; - private ArrayList getWorkers(int threads, String[] queries){ - ArrayList workers = new ArrayList(); - HashMap workerConfig = new HashMap(); + private List> getWorkers(int threads, String[] queries) { + List> workers = new ArrayList<>(); + Map workerConfig = new HashMap<>(); workerConfig.put("className", MockupWorker.class.getCanonicalName()); - workerConfig.put("queries", queries); + workerConfig.put("stringQueries", queries); workerConfig.put("threads", threads); workers.add(workerConfig); return workers; } - private Connection getConnection(){ + private Connection getConnection() { Connection con = new Connection(); con.setName("test"); con.setEndpoint("test/sparql"); return con; } - private LinkedHashMap getQueryHandler(){ - LinkedHashMap queryHandler = new LinkedHashMap(); - queryHandler.put("className", "InstancesQueryHandler"); - return queryHandler; - } - private void init(){ StorageManager storageManager = StorageManager.getInstance(); MetricManager mmanger = MetricManager.getInstance(); @@ -64,26 +58,24 @@ private void init(){ @Test public void checkStresstestNoQM() throws IOException { - Stresstest task = new Stresstest( getWorkers(2, queries), getQueryHandler(), 10); - task.qhCacheFolder=UUID.randomUUID().toString(); + Stresstest task = new Stresstest(getWorkers(2, this.queries), 10); + task.cacheFolder = UUID.randomUUID().toString(); task.init(new String[]{"1", "1/1", "1/1/1"}, "test", getConnection()); init(); - Instant start = Instant.now(); task.execute(); - Instant end = Instant.now(); //2 queries in mix, 10 executions on 2 workers -> 40 queries assertEquals(40, task.getExecutedQueries()); - FileUtils.deleteDirectory(new File(task.qhCacheFolder)); + FileUtils.deleteDirectory(new File(task.cacheFolder)); } @Test public void checkStresstestTL() throws IOException { - Stresstest task = new Stresstest(5000, getWorkers(2, queries), getQueryHandler()); - task.qhCacheFolder=UUID.randomUUID().toString(); + Stresstest task = new Stresstest(5000, getWorkers(2, this.queries)); + task.cacheFolder = UUID.randomUUID().toString(); task.init(new String[]{"1", "1/1", "1/1/1"}, "test", getConnection()); @@ -93,70 +85,69 @@ public void checkStresstestTL() throws IOException { task.execute(); Instant end = Instant.now(); //give about 200milliseconds time for init and end stuff - assertEquals(5000.0, end.toEpochMilli()-start.toEpochMilli(), 300.0); - FileUtils.deleteDirectory(new File(task.qhCacheFolder)); + assertEquals(5000.0, end.toEpochMilli() - start.toEpochMilli(), 300.0); + FileUtils.deleteDirectory(new File(task.cacheFolder)); } @Test public void warmupTest() throws IOException { //check if not executing - Stresstest task = new Stresstest(5000, getWorkers(2, queries), getQueryHandler()); - task.qhCacheFolder=UUID.randomUUID().toString(); + Stresstest task = new Stresstest(5000, getWorkers(2, this.queries)); + task.cacheFolder = UUID.randomUUID().toString(); task.init(new String[]{"1", "1/1", "1/1/1"}, "test", getConnection()); Instant start = Instant.now(); assertEquals(0, task.warmup()); Instant end = Instant.now(); - assertEquals(0.0, end.toEpochMilli()-start.toEpochMilli(), 5.0); + assertEquals(0.0, end.toEpochMilli() - start.toEpochMilli(), 5.0); //check if executing - LinkedHashMap warmup = new LinkedHashMap(); - warmup.put("workers", getWorkers(2, queries)); + Map warmup = new LinkedHashMap<>(); + warmup.put("workers", getWorkers(2, this.queries)); warmup.put("timeLimit", 350); - FileUtils.deleteDirectory(new File(task.qhCacheFolder)); + FileUtils.deleteDirectory(new File(task.cacheFolder)); - task = new Stresstest(5000, getWorkers(2, queries), getQueryHandler(), warmup); - task.qhCacheFolder=UUID.randomUUID().toString(); + task = new Stresstest(5000, getWorkers(2, this.queries), warmup); + task.cacheFolder = UUID.randomUUID().toString(); task.init(new String[]{"1", "1/1", "1/1/1"}, "test", getConnection()); start = Instant.now(); long queriesExecuted = task.warmup(); end = Instant.now(); // might sadly be 400 or 500 as the warmup works in 100th steps, also overhead, as long as executed Queries are 6 its fine - assertEquals(350.0, end.toEpochMilli()-start.toEpochMilli(), 250.0); + assertEquals(350.0, end.toEpochMilli() - start.toEpochMilli(), 250.0); //each worker could execute 3 query assertEquals(6, queriesExecuted); - FileUtils.deleteDirectory(new File(task.qhCacheFolder)); + FileUtils.deleteDirectory(new File(task.cacheFolder)); } @Test public void workerCreationTest() throws IOException { - ArrayList worker = getWorkers(2, queries); - worker.addAll(getWorkers(1, queries2)); - Stresstest task = new Stresstest(5000, worker, getQueryHandler()); - task.qhCacheFolder=UUID.randomUUID().toString(); + List> worker = getWorkers(2, this.queries); + worker.addAll(getWorkers(1, this.queries2)); + Stresstest task = new Stresstest(5000, worker); + task.cacheFolder = UUID.randomUUID().toString(); task.init(new String[]{"1", "1/1", "1/1/1"}, "test", getConnection()); List workers = task.workers; assertEquals(3, workers.size()); - int q1=0; - int q2=0; + int q1 = 0; + int q2 = 0; // alittle bit hacky but should be sufficient - for(Worker w : workers){ - MockupWorker mockupWorker = (MockupWorker)w; + for (Worker w : workers) { + MockupWorker mockupWorker = (MockupWorker) w; String[] queries = mockupWorker.getStringQueries(); - if(queries.hashCode()==this.queries.hashCode()){ + if (Arrays.hashCode(queries) == Arrays.hashCode(this.queries)) { q1++; - } - else if(queries.hashCode()==this.queries2.hashCode()){ + } else if (Arrays.hashCode(queries) == Arrays.hashCode(this.queries2)) { q2++; } } assertEquals(2, q1); assertEquals(1, q2); - FileUtils.deleteDirectory(new File(task.qhCacheFolder)); + FileUtils.deleteDirectory(new File(task.cacheFolder)); } } diff --git a/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/worker/MockupWorker.java b/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/worker/MockupWorker.java index 4cdcff054..08c3e0476 100644 --- a/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/worker/MockupWorker.java +++ b/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/worker/MockupWorker.java @@ -4,37 +4,37 @@ import org.aksw.iguana.cc.model.QueryExecutionStats; import org.aksw.iguana.commons.annotation.Nullable; -import java.io.IOException; +import java.util.HashMap; +import java.util.Map; public class MockupWorker extends AbstractWorker { private int counter = 0; - private String[] queries = new String[]{ + private final String[] queries; - }; - - public MockupWorker(String[] queries, Integer workerID, @Nullable Integer timeLimit, Connection connection, String taskID) { - super(taskID, connection, "src/test/resources/mockupq.txt", 0, timeLimit, 0, 0, workerID); - this.queries = queries; - } - - - public MockupWorker(String taskID, Connection connection, String queriesFile, Integer timeOut, Integer timeLimit, Integer fixedLatency, Integer gaussianLatency, Integer workerID) { - super(taskID, connection, queriesFile, timeOut, timeLimit, fixedLatency, gaussianLatency, workerID); + public MockupWorker(String[] stringQueries, Integer workerID, @Nullable Integer timeLimit, Connection connection, String taskID) { + super(taskID, workerID, connection, getQueryConfig(), 0, timeLimit, 0, 0); + this.queries = stringQueries; } public String[] getStringQueries() { return queries; } + private static Map getQueryConfig() { + Map queryConfig = new HashMap<>(); + queryConfig.put("location", "src/test/resources/mockupq.txt"); + return queryConfig; + } + @Override public void executeQuery(String query, String queryID) { QueryExecutionStats results = new QueryExecutionStats(); - long execTime = workerID * 10 + 100; + long execTime = this.workerID * 10 + 100; try { Thread.sleep(execTime); results.setResponseCode(200); - results.setResultSize(workerID*100+100); + results.setResultSize(this.workerID * 100 + 100); } catch (InterruptedException e) { e.printStackTrace(); results.setResponseCode(400); @@ -46,12 +46,12 @@ public void executeQuery(String query, String queryID) { } @Override - public void getNextQuery(StringBuilder queryStr, StringBuilder queryID) throws IOException { - if(counter>=queries.length){ - counter=0; + public void getNextQuery(StringBuilder queryStr, StringBuilder queryID) { + if (this.counter >= this.queries.length) { + this.counter = 0; } - queryStr.append(queries[counter]); - queryID.append("query").append(counter); - counter++; + queryStr.append(this.queries[this.counter]); + queryID.append("query").append(this.counter); + this.counter++; } } From 6b5a7bea1636293fc6f0f7fb98d372b15e4e3311 Mon Sep 17 00:00:00 2001 From: frensing Date: Fri, 4 Nov 2022 20:44:48 +0200 Subject: [PATCH 18/47] PatternHandler and remove of old QueryHandler --- .../iguana/cc/controller/TaskController.java | 3 +- .../cc/query/AbstractWorkerQueryHandler.java | 80 ------ .../aksw/iguana/cc/query/QueryHandler.java | 32 --- .../iguana/cc/query/QueryHandlerFactory.java | 15 -- .../iguana/cc/query/handler/QueryHandler.java | 7 +- .../cc/query/impl/InstancesQueryHandler.java | 194 --------------- .../cc/query/impl/PatternQueryHandler.java | 233 ------------------ .../cc/query/pattern/PatternHandler.java | 159 ++++++++++++ .../cc/query/set/impl/FileBasedQuerySet.java | 40 +-- .../cc/query/set/impl/InMemQuerySet.java | 38 +-- .../query/set/newimpl/FileBasedQuerySet.java | 18 -- .../cc/query/set/newimpl/InMemQuerySet.java | 39 --- .../source/impl/FileSeparatorQuerySource.java | 2 - .../aksw/iguana/cc/tasks/impl/Stresstest.java | 7 - .../aksw/iguana/cc/worker/AbstractWorker.java | 24 -- .../impl/PatternBasedQueryHandlerTest.java | 187 -------------- .../pattern/PatternBasedQueryHandlerTest.java | 141 +++++++++++ .../PatternQueryHandlerTest.java | 80 +++--- .../iguana/cc/tasks/impl/StresstestTest.java | 21 +- 19 files changed, 370 insertions(+), 950 deletions(-) delete mode 100644 iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/AbstractWorkerQueryHandler.java delete mode 100644 iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/QueryHandler.java delete mode 100644 iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/QueryHandlerFactory.java delete mode 100644 iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/impl/InstancesQueryHandler.java delete mode 100644 iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/impl/PatternQueryHandler.java create mode 100644 iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/pattern/PatternHandler.java delete mode 100644 iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/newimpl/FileBasedQuerySet.java delete mode 100644 iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/newimpl/InMemQuerySet.java delete mode 100644 iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/impl/PatternBasedQueryHandlerTest.java create mode 100644 iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/pattern/PatternBasedQueryHandlerTest.java rename iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/{impl => pattern}/PatternQueryHandlerTest.java (68%) diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/controller/TaskController.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/controller/TaskController.java index 33483b144..b0e589abc 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/controller/TaskController.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/controller/TaskController.java @@ -18,8 +18,7 @@ */ public class TaskController { - private static final Logger LOGGER = LoggerFactory - .getLogger(TaskController.class); + private static final Logger LOGGER = LoggerFactory.getLogger(TaskController.class); public void startTask(String[] ids, String dataset, Connection con, Task task) { TaskManager tmanager = new TaskManager(); diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/AbstractWorkerQueryHandler.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/AbstractWorkerQueryHandler.java deleted file mode 100644 index 4361c84b1..000000000 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/AbstractWorkerQueryHandler.java +++ /dev/null @@ -1,80 +0,0 @@ -package org.aksw.iguana.cc.query; - -import org.aksw.iguana.cc.query.set.QuerySet; -import org.aksw.iguana.cc.worker.AbstractWorker; -import org.aksw.iguana.cc.worker.Worker; -import org.aksw.iguana.cc.worker.impl.UPDATEWorker; - -import java.io.File; -import java.util.Collection; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; - -/** - * - * An abstract class to use if the QueryHandler should work with Workers. (e.g. in the Stresstest Task) - * - * @author f.conrads - * - */ -public abstract class AbstractWorkerQueryHandler implements QueryHandler{ - - /** - * Will contain the path of the worker specified query files to - * the Files where the final querys will be saved - */ - private Map mapping = new HashMap(); - private HashSet sparqlKeys = new HashSet(); - private HashSet updateKeys = new HashSet(); - private Collection workers; - - /** - * - * @param workers - */ - public AbstractWorkerQueryHandler(Collection workers) { - this.workers = workers; - for(Worker worker : workers) { - if(worker instanceof UPDATEWorker) { - updateKeys.add(((UPDATEWorker)worker).getQueriesFileName()); - } else { - sparqlKeys.add(((AbstractWorker)worker).getQueriesFileName()); - } - } - } - - @Override - public Map generate() { - for(String sparqlKey : sparqlKeys) { - mapping.put(sparqlKey, generateQueries(sparqlKey)); - } - for(String updateKey : updateKeys) { - mapping.put(updateKey, generateUPDATE(updateKey)); - } - for(Worker worker : workers) { - if(worker instanceof AbstractWorker) { - ((AbstractWorker)worker).setQueriesList( - mapping.get(((AbstractWorker)worker).getQueriesFileName())); - } - } - return mapping; - } - - /** - * This method will generate SPARQL Queries given a file with queries. - * - * @param queryFileName The queries file - * @return for each query in the file, a File representing the query - */ - protected abstract QuerySet[] generateQueries(String queryFileName) ; - - /** - * This method will generate UPDATE Queries given a folder with files in which updates are stated. - * - * @param updatePath The path to the updates - * @return for each update, a File representing it. - */ - protected abstract QuerySet[] generateUPDATE(String updatePath) ; - -} diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/QueryHandler.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/QueryHandler.java deleted file mode 100644 index ab79141fd..000000000 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/QueryHandler.java +++ /dev/null @@ -1,32 +0,0 @@ -package org.aksw.iguana.cc.query; - -import org.aksw.iguana.cc.query.set.QuerySet; -import org.apache.jena.rdf.model.Model; - -import java.io.File; -import java.util.Map; - -/** - * The QueryHandler interface - *
- * The QueryHandler can be used to generate queries in the Tasks. - * - * @author f.conrads - * - */ -public interface QueryHandler { - - /** - * This will generate the queries. - * @return - */ - Map generate(); - - /** - * Generates some stats for the queries - * @param taskID - * @return - */ - Model generateTripleStats(String taskID); - -} diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/QueryHandlerFactory.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/QueryHandlerFactory.java deleted file mode 100644 index 3014d6607..000000000 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/QueryHandlerFactory.java +++ /dev/null @@ -1,15 +0,0 @@ -package org.aksw.iguana.cc.query; - -import org.aksw.iguana.commons.factory.TypedFactory; - - -/** - * Factory to create a QueryHandler based upon a class name and constructor arguments - * - * @author f.conrads - * - */ -public class QueryHandlerFactory extends TypedFactory { - - -} diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/handler/QueryHandler.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/handler/QueryHandler.java index 598e89b90..5fc13af10 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/handler/QueryHandler.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/handler/QueryHandler.java @@ -6,8 +6,8 @@ import org.aksw.iguana.cc.query.selector.impl.LinearQuerySelector; import org.aksw.iguana.cc.query.selector.impl.RandomQuerySelector; import org.aksw.iguana.cc.query.set.QuerySet; -import org.aksw.iguana.cc.query.set.newimpl.FileBasedQuerySet; -import org.aksw.iguana.cc.query.set.newimpl.InMemQuerySet; +import org.aksw.iguana.cc.query.set.impl.FileBasedQuerySet; +import org.aksw.iguana.cc.query.set.impl.InMemQuerySet; import org.aksw.iguana.cc.query.source.QuerySource; import org.aksw.iguana.cc.query.source.impl.FileLineQuerySource; import org.aksw.iguana.cc.query.source.impl.FileSeparatorQuerySource; @@ -40,14 +40,11 @@ public class QueryHandler { protected LanguageProcessor langProcessor; - protected String outputFolder; - public QueryHandler(Map config, Integer workerID) { this.config = config; this.workerID = workerID; this.location = (String) config.get("location"); - this.outputFolder = (String) config.get("outputFolder"); initQuerySet(); initQuerySelector(); diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/impl/InstancesQueryHandler.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/impl/InstancesQueryHandler.java deleted file mode 100644 index ee06448f0..000000000 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/impl/InstancesQueryHandler.java +++ /dev/null @@ -1,194 +0,0 @@ -package org.aksw.iguana.cc.query.impl; - -import org.aksw.iguana.cc.lang.LanguageProcessor; -import org.aksw.iguana.cc.lang.QueryWrapper; -import org.aksw.iguana.cc.lang.impl.SPARQLLanguageProcessor; -import org.aksw.iguana.cc.query.AbstractWorkerQueryHandler; -import org.aksw.iguana.cc.query.set.QuerySet; -import org.aksw.iguana.cc.query.set.impl.FileBasedQuerySet; -import org.aksw.iguana.cc.query.set.impl.InMemQuerySet; -import org.aksw.iguana.cc.utils.FileUtils; -import org.aksw.iguana.cc.utils.SPARQLQueryStatistics; -import org.aksw.iguana.cc.worker.Worker; -import org.aksw.iguana.commons.annotation.Shorthand; -import org.aksw.iguana.commons.factory.TypedFactory; -import org.apache.jena.ext.com.google.common.collect.Lists; -import org.apache.jena.rdf.model.Model; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.*; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; - -/** - * - * A QueryHandler for already instances of queries. - * - * @author f.conrads - * - */ -@Shorthand("InstancesQueryHandler") -public class InstancesQueryHandler extends AbstractWorkerQueryHandler { - - private static final Logger LOGGER = LoggerFactory.getLogger(InstancesQueryHandler.class); - - protected String outputFolder = "queryCache"; - - protected HashMap type2IDcounter = new HashMap(); - - protected SPARQLQueryStatistics qs = new SPARQLQueryStatistics(); - - private QuerySet[] queryFiles; - - protected LanguageProcessor langProcessor = new SPARQLLanguageProcessor(); - protected int hashcode; - - //protected int hashcode; - - /** - * Default Constructor - * - * @param workers Workers to consider queryFiles/updatePaths of - */ - public InstancesQueryHandler(List workers) { - super(workers); - } - - public InstancesQueryHandler(List workers, String lang) { - super(workers); - langProcessor = new TypedFactory().create(lang, new HashMap()); - } - - @Override - protected QuerySet[] generateQueries(String queryFileName) { - // Save hashcode of the file content for later use in generating stats - hashcode = FileUtils.getHashcodeFromFileContent(queryFileName); - - QuerySet[] queries = generateQueryPerLine(queryFileName, langProcessor.getQueryPrefix(), hashcode); - this.queryFiles = queries; - - - return queries; - } - - protected QuerySet[] generateQueryPerLine(String queryFileName, String idPrefix, int hashcode) { - File queryFile = new File(queryFileName); - List ret = new LinkedList(); - LOGGER.info("[QueryHandler: {{}}] Queries will now be instantiated", this.getClass().getName()); - - try (BufferedReader reader = new BufferedReader(new FileReader(queryFileName))) { - String queryStr; - int id=0; - while ((queryStr = reader.readLine()) != null) { - if (queryStr.isEmpty()) { - continue; - } - ret.add(new InMemQuerySet(idPrefix+id++, getInstances(queryStr))); - - } - } catch (IOException e) { - LOGGER.error("could not read queries"); - } - LOGGER.info("[QueryHandler: {{}}] Finished instantiation of queries", this.getClass().getName()); - return ret.toArray(new QuerySet[]{}); - - } - - protected List getInstances(String queryStr) { - return Lists.newArrayList(queryStr); - } - - - protected File createFileWithID(File rootFolder, String idPrefix) throws IOException { - // create a File with an ID - int id = 0; - if (type2IDcounter.containsKey(idPrefix)) { - id = type2IDcounter.get(idPrefix); - } - File out = new File(rootFolder.getAbsolutePath() + File.separator + idPrefix + id); - out.createNewFile(); - id++; - type2IDcounter.put(idPrefix, id); - return out; - } - - @Override - protected QuerySet[] generateUPDATE(String updatePath) { - File dir = new File(updatePath); - if (dir.exists()) { - if (dir.isDirectory()) { - LOGGER.info("[QueryHandler: {{}}] Uses all UPDATE files in {{}}", this.getClass().getName(), - updatePath); - // dir is directory, get all files in the folder - File[] files = dir.listFiles(); - QuerySet[] sets = new QuerySet[files.length]; - for(int i=0;i ret = new LinkedList(); - LOGGER.info("[QueryHandler: {{}}] Queries will now be instantiated", this.getClass().getName()); - - try (BufferedReader reader = new BufferedReader(new FileReader(updatePath))) { - String queryStr; - int id=0; - while ((queryStr = reader.readLine()) != null) { - if (queryStr.isEmpty()) { - continue; - } - ret.add(new InMemQuerySet(idPrefix+id++, Lists.newArrayList(queryStr))); - - } - } catch (IOException e) { - LOGGER.error("could not read queries"); - } - LOGGER.info("[QueryHandler: {{}}] Finished instantiation of queries", this.getClass().getName()); - return ret.toArray(new QuerySet[]{}); - } - - @Override - public Model generateTripleStats(String taskID) { - List queries = new ArrayList(); - for (QuerySet queryFile : queryFiles) { - try { - String query = queryFile.getQueryAtPos(0); - queries.add(new QueryWrapper(query, queryFile.getName())); - }catch(IOException e){ - LOGGER.error("[QueryHandler: {{}}] Cannot read file {{}}", this.getClass().getName(), - queryFile.getName()); - } - } - return langProcessor.generateTripleStats(queries, hashcode+"", taskID); - } - - - public void setOutputFolder(String outputFolder) { - this.outputFolder = outputFolder; - } -} diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/impl/PatternQueryHandler.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/impl/PatternQueryHandler.java deleted file mode 100644 index 1d14b046f..000000000 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/impl/PatternQueryHandler.java +++ /dev/null @@ -1,233 +0,0 @@ -package org.aksw.iguana.cc.query.impl; - -import org.aksw.iguana.cc.query.set.QuerySet; -import org.aksw.iguana.cc.query.set.impl.FileBasedQuerySet; -import org.aksw.iguana.cc.worker.Worker; -import org.aksw.iguana.commons.annotation.Shorthand; -import org.apache.jena.query.*; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.*; -import java.util.*; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -/** - * This QueryHandler uses query patterns and converts them into query instances.
- * A query pattern is a SPARQL 1.1 Query which can have additional variables %%var[0/9]+%% in the - * Basic Graph Pattern.

- * For example: SELECT * {?s %%var1%% ?o . ?o <http://exa.com> %%var100%%}
- * This QueryHandler will then convert this query to:
- * SELECT ?var1 ?var100 {?s ?var1 ?o . ?o <http://exa.com> ?var100}
- * and will request query solutions from a user given sparql endpoint (e.g DBpedia)
- * The solution will then be instantiated into the query pattern. - * The result can look like follows:

- * SELECT * {?s <http://prop/1> ?o . ?o <http://exa.com> "123"}
- * SELECT * {?s <http://prop/1> ?o . ?o <http://exa.com> "12"}
- * SELECT * {?s <http://prop/2> ?o . ?o <http://exa.com> "1234"}
- * - * - * @author f.conrads - * - */ -@Shorthand("PatternQueryHandler") -public class PatternQueryHandler extends InstancesQueryHandler { - - private static final Logger LOGGER = LoggerFactory - .getLogger(PatternQueryHandler.class); - - private String service; - - private Long limit = 2000l; - - /** - * - * The constructor for the pattern based QueryHandler.
- * Query Instances will be restricted to 2000 per QueryPattern
- *
- * This will check all Workers if they are of type SPARQL or UPDATEWorker and gets their - * querysFile/updatePath to generate queries out of it. - * - * @param workers - * @param endpoint the sparql endpoint to derive the variable instances - */ - public PatternQueryHandler(List workers, String endpoint) { - super(workers); - this.service = endpoint; - } - - /** - * - * The constructor for the pattern based QueryHandler.
- *
- * This will check all Workers if they are of type SPARQL or UPDATEWorker and gets their - * querysFile/updatePath to generate queries out of it. - * - * @param workers - * @param endpoint the sparql endpoint to derive the variable instances - * @param limit the resitriction of query instances per query pattern as String - */ - public PatternQueryHandler(LinkedList workers, String endpoint, String limit) { - this(workers, endpoint, Long.parseLong(limit)); - } - - /** - * - * The constructor for the pattern based QueryHandler.
- *
- * This will check all Workers if they are of type SPARQL or UPDATEWorker and gets their - * querysFile/updatePath to generate queries out of it. - * - * @param workers - * @param endpoint the sparql endpoint to derive the variable instances - * @param limit the resitriction of query instances per query pattern - */ - public PatternQueryHandler(LinkedList workers, String endpoint, Long limit) { - super(workers); - this.service = endpoint; - this.limit = limit; - } - - - - protected String replaceVars(String queryStr, Set varNames) { - String command = queryStr; - Pattern pattern = Pattern.compile("%%var[0-9]+%%"); - Matcher m = pattern.matcher(queryStr); - while(m.find()) { - String eVar = m.group(); - String var = eVar.replace("%", ""); - command = command.replace(eVar, "?"+var); - varNames.add(var); - } - return command; - } - - - @Override - protected QuerySet[] generateQueryPerLine(String queryFileName, String idPrefix, int hashcode) { - File queryFile = new File(queryFileName); - List ret = new LinkedList(); - // check if folder is cached - if (queryFile.exists()) { - File outputFolder = new File(this.outputFolder + File.separator + hashcode); - if (outputFolder.exists()) { - LOGGER.warn("[QueryHandler: {{}}] queries were instantiated already, will use old instances. To generate them new remove the {{}} folder", - this.getClass().getName(), this.outputFolder + File.separator + hashcode); - // is cached use caching - for(File f : outputFolder.listFiles()){ - try { - ret.add(new FileBasedQuerySet(f)); - } catch (IOException e) { - e.printStackTrace(); - } - } - return ret.toArray(new QuerySet[]{}); - } else { - LOGGER.info("[QueryHandler: {{}}] Queries will now be instantiated", this.getClass().getName()); - // create directorys - outputFolder.mkdirs(); - try (BufferedReader reader = new BufferedReader(new FileReader(queryFileName))) { - String queryStr; - // iterate over all queries - while ((queryStr = reader.readLine()) != null) { - if (queryStr.isEmpty()) { - continue; - } - //create file with id and write query to it - File out = createFileWithID(outputFolder, idPrefix); - try (PrintWriter pw = new PrintWriter(out)) { - for (String query : getInstances(queryStr)) { - pw.println(query); - } - } - QuerySet qs = new FileBasedQuerySet(out); - ret.add(qs); - - } - } catch (IOException e) { - LOGGER.error("[QueryHandler: {{}}] could not write instances to folder {{}}", - this.getClass().getName(), outputFolder.getAbsolutePath()); - } - LOGGER.info("[QueryHandler: {{}}] Finished instantiation of queries", this.getClass().getName()); - } - return ret.toArray(new QuerySet[]{}); - } else { - LOGGER.error("[QueryHandler: {{}}] Queries with file {{}} could not be instantiated due to missing file", - this.getClass().getName(), queryFileName); - } - return new QuerySet[]{}; - } - - @Override - protected List getInstances(String queryStr) { - List instances = new ArrayList<>(); - - //check if query is already an instance - try{ - QueryFactory.create(queryStr); - //query is instance - LOGGER.debug("[QueryHandler: {{}}] Query is already an instance: {{}}", this.getClass().getName(), queryStr); - instances.add(queryStr); - return instances; - }catch(Exception e) { - //query is pattern, nothing to do - } - - //get vars from queryStr - Set varNames = new HashSet(); - String command = replaceVars(queryStr, varNames); - - //generate parameterized sparql query - ParameterizedSparqlString pss = new ParameterizedSparqlString(); - pss.setCommandText(command); - ResultSet exchange = getInstanceVars(pss, varNames); - // exchange vars in PSS - if(!exchange.hasNext()) { - //no solution - LOGGER.warn("[QueryHandler: {{}}] Query has no solution, will use variables instead of var instances: {{}}", this.getClass().getName(), queryStr); - instances.add(command); - } - while(exchange.hasNext()) { - QuerySolution solution = exchange.next(); - for(String var : exchange.getResultVars()) { - //exchange variable with - pss.clearParam(var); - pss.setParam(var, solution.get(var)); - } - instances.add(pss.toString()); - } - LOGGER.debug("Found instances {}", instances); - - return instances; - } - - - protected ResultSet getInstanceVars(ParameterizedSparqlString pss, Set varNames) { - QueryExecution exec = QueryExecutionFactory.createServiceRequest(service, convertToSelect(pss,varNames)); - //return result set - return exec.execSelect(); - } - - protected Query convertToSelect(ParameterizedSparqlString pss, Set varNames) { - if(varNames.isEmpty()){ - return pss.asQuery(); - } - Query queryCpy = pss.asQuery(); - queryCpy.getQueryPattern(); - - StringBuilder queryStr = new StringBuilder("SELECT DISTINCT "); - for(String varName : varNames) { - queryStr.append("?").append(varName).append(" "); - } - queryStr.append(queryCpy.getQueryPattern()); - ParameterizedSparqlString pssSelect = new ParameterizedSparqlString(); - pssSelect.setCommandText(queryStr.toString()); - pssSelect.setNsPrefixes(pss.getNsPrefixMap()); - pssSelect.append(" LIMIT "); - pssSelect.append(this.limit); - return pssSelect.asQuery(); - } - -} diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/pattern/PatternHandler.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/pattern/PatternHandler.java new file mode 100644 index 000000000..371deb06a --- /dev/null +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/pattern/PatternHandler.java @@ -0,0 +1,159 @@ +package org.aksw.iguana.cc.query.pattern; + +import org.aksw.iguana.cc.query.source.QuerySource; +import org.aksw.iguana.cc.query.source.impl.FileLineQuerySource; +import org.apache.jena.query.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.PrintWriter; +import java.util.*; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class PatternHandler { + private static final Logger LOGGER = LoggerFactory.getLogger(PatternHandler.class); + + private final Map config; + private final QuerySource querySource; + private String endpoint; + private Long limit; + private String outputFolder; + + public PatternHandler(Map config, QuerySource querySource) { + this.config = config; + this.querySource = querySource; + + init(); + } + + public QuerySource generateQuerySource() { + File cacheFile = new File(this.outputFolder + File.separator + this.querySource.getHashcode()); + if (cacheFile.exists()) { + + LOGGER.warn("Output folder already exists. Will not generate queries again. To generate them new remove the {{}} folder", cacheFile.getAbsolutePath()); + + } else { + LOGGER.info("Generating queries for pattern queries"); + File outFolder = new File(this.outputFolder); + if (!outFolder.exists()) { + outFolder.mkdirs(); + } + + try (PrintWriter pw = new PrintWriter(cacheFile)) { + for (int i = 0; i < this.querySource.size(); i++) { + for (String query : generateQueries(this.querySource.getQuery(i))) { + pw.println(query); + } + } + } catch (Exception e) { + LOGGER.error("Could not write to file", e); + } + } + + return new FileLineQuerySource(cacheFile.getAbsolutePath()); + } + + private void init() { + this.endpoint = (String) this.config.get("endpoint"); + if (this.endpoint == null) { + LOGGER.error("No endpoint given for pattern handler"); + } + + this.outputFolder = (String) this.config.getOrDefault("outputFolder", "queryCache"); + + Object limitObj = this.config.getOrDefault("limit", 2000L); + if (limitObj instanceof Number) { + this.limit = ((Number) limitObj).longValue(); + } else if (limitObj instanceof String) { + this.limit = Long.parseLong((String) limitObj); + } else { + LOGGER.error("could not parse limit"); + } + } + + protected List generateQueries(String query) { + List queries = new LinkedList<>(); + + try { + // if query is already an instance, we do not need to generate anything + QueryFactory.create(query); + LOGGER.debug("Query is already an instance: {{}}", query); + queries.add(query); + return queries; + } catch (Exception ignored) { + } + + //get vars from queryStr + Set varNames = new HashSet<>(); + String command = replaceVars(query, varNames); + + //generate parameterized sparql query + ParameterizedSparqlString pss = new ParameterizedSparqlString(); + pss.setCommandText(command); + ResultSet exchange = getInstanceVars(pss, varNames); + // exchange vars in PSS + if (!exchange.hasNext()) { + //no solution + LOGGER.warn("Query has no solution, will use variables instead of var instances: {{}}", query); + queries.add(command); + } + while (exchange.hasNext()) { + QuerySolution solution = exchange.next(); + for (String var : exchange.getResultVars()) { + //exchange variable with + pss.clearParam(var); + pss.setParam(var, solution.get(var)); + } + queries.add(pss.toString()); + } + LOGGER.debug("Found instances {}", queries); + + return queries; + } + + protected String replaceVars(String queryStr, Set varNames) { + String command = queryStr; + Pattern pattern = Pattern.compile("%%var[0-9]+%%"); + Matcher m = pattern.matcher(queryStr); + while (m.find()) { + String eVar = m.group(); + String var = eVar.replace("%", ""); + command = command.replace(eVar, "?" + var); + varNames.add(var); + } + return command; + } + + protected ResultSet getInstanceVars(ParameterizedSparqlString pss, Set varNames) { + QueryExecution exec = QueryExecutionFactory.createServiceRequest(this.endpoint, convertToSelect(pss, varNames)); + //return result set + return exec.execSelect(); + } + + protected Query convertToSelect(ParameterizedSparqlString pss, Set varNames) { + Query queryCpy; + try { + if (varNames.isEmpty()) { + return pss.asQuery(); + } + queryCpy = pss.asQuery(); + } catch (Exception e) { + LOGGER.error("Could not convert query to select (is it update query?): {{}}", pss.toString(), e); + return null; + } + + StringBuilder queryStr = new StringBuilder("SELECT DISTINCT "); + for (String varName : varNames) { + queryStr.append("?").append(varName).append(" "); + } + queryStr.append(queryCpy.getQueryPattern()); + ParameterizedSparqlString pssSelect = new ParameterizedSparqlString(); + pssSelect.setCommandText(queryStr.toString()); + pssSelect.setNsPrefixes(pss.getNsPrefixMap()); + pssSelect.append(" LIMIT "); + pssSelect.append(this.limit); + return pssSelect.asQuery(); + } +} diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/impl/FileBasedQuerySet.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/impl/FileBasedQuerySet.java index 470ef7382..09a1cb24b 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/impl/FileBasedQuerySet.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/impl/FileBasedQuerySet.java @@ -1,46 +1,18 @@ package org.aksw.iguana.cc.query.set.impl; -import org.aksw.iguana.cc.query.set.QuerySet; -import org.aksw.iguana.cc.utils.FileUtils; +import org.aksw.iguana.cc.query.set.AbstractQuerySet; +import org.aksw.iguana.cc.query.source.QuerySource; -import java.io.File; import java.io.IOException; -/** - * File based query set - */ -public class FileBasedQuerySet implements QuerySet { +public class FileBasedQuerySet extends AbstractQuerySet { - private final File queryFile; - private int size; - - - public FileBasedQuerySet(File queryFile) throws IOException { - this.queryFile = queryFile; - size = FileUtils.countLines(queryFile); - } - - public File getFile() { - return queryFile; + public FileBasedQuerySet(String name, QuerySource querySource) { + super(name, querySource); } @Override public String getQueryAtPos(int pos) throws IOException { - return FileUtils.readLineAt(pos, queryFile); - } - - @Override - public int size() { - return size; - } - - @Override - public String getName() { - return queryFile.getName(); - } - - @Override - public int getHashcode() { - return queryFile.hashCode(); + return this.querySource.getQuery(pos); } } diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/impl/InMemQuerySet.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/impl/InMemQuerySet.java index 3a71b63a5..216a84375 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/impl/InMemQuerySet.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/impl/InMemQuerySet.java @@ -1,37 +1,39 @@ package org.aksw.iguana.cc.query.set.impl; -import org.aksw.iguana.cc.query.set.QuerySet; +import org.aksw.iguana.cc.query.set.AbstractQuerySet; +import org.aksw.iguana.cc.query.source.QuerySource; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.IOException; import java.util.List; -public class InMemQuerySet implements QuerySet { +public class InMemQuerySet extends AbstractQuerySet { + + private static final Logger LOGGER = LoggerFactory.getLogger(InMemQuerySet.class); private List queries; - private String name; - public InMemQuerySet(String queryID, List queries){ - name=queryID; - this.queries=queries; + public InMemQuerySet(String name, QuerySource querySource) { + super(name, querySource); + loadQueries(); } - @Override - public String getQueryAtPos(int pos) throws IOException { - return queries.get(pos); + private void loadQueries() { + try { + this.queries = this.querySource.getAllQueries(); + } catch (IOException e) { + LOGGER.error("Could not read queries"); + } } @Override - public int size() { - return queries.size(); + public String getQueryAtPos(int pos) { + return this.queries.get(pos); } @Override - public String getName() { - return name; - } - - @Override - public int getHashcode() { - return queries.hashCode(); + public int size() { + return this.queries.size(); } } diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/newimpl/FileBasedQuerySet.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/newimpl/FileBasedQuerySet.java deleted file mode 100644 index b819e4fb6..000000000 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/newimpl/FileBasedQuerySet.java +++ /dev/null @@ -1,18 +0,0 @@ -package org.aksw.iguana.cc.query.set.newimpl; - -import org.aksw.iguana.cc.query.set.AbstractQuerySet; -import org.aksw.iguana.cc.query.source.QuerySource; - -import java.io.IOException; - -public class FileBasedQuerySet extends AbstractQuerySet { - - public FileBasedQuerySet(String name, QuerySource querySource) { - super(name, querySource); - } - - @Override - public String getQueryAtPos(int pos) throws IOException { - return this.querySource.getQuery(pos); - } -} diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/newimpl/InMemQuerySet.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/newimpl/InMemQuerySet.java deleted file mode 100644 index f1dbc35f1..000000000 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/newimpl/InMemQuerySet.java +++ /dev/null @@ -1,39 +0,0 @@ -package org.aksw.iguana.cc.query.set.newimpl; - -import org.aksw.iguana.cc.query.set.AbstractQuerySet; -import org.aksw.iguana.cc.query.source.QuerySource; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.util.List; - -public class InMemQuerySet extends AbstractQuerySet { - - private static final Logger LOGGER = LoggerFactory.getLogger(InMemQuerySet.class); - - private List queries; - - public InMemQuerySet(String name, QuerySource querySource) { - super(name, querySource); - loadQueries(); - } - - private void loadQueries() { - try { - this.queries = this.querySource.getAllQueries(); - } catch (IOException e) { - LOGGER.error("Could not read queries"); - } - } - - @Override - public String getQueryAtPos(int pos) throws IOException { - return this.queries.get(pos); - } - - @Override - public int size() { - return this.queries.size(); - } -} diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/impl/FileSeparatorQuerySource.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/impl/FileSeparatorQuerySource.java index 572d5bdd9..7f2756019 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/impl/FileSeparatorQuerySource.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/impl/FileSeparatorQuerySource.java @@ -78,8 +78,6 @@ public String getQuery(int index) throws IOException { } catch (FileNotFoundException e) { LOGGER.error("Could not read queries"); } - - // todo return null; } diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/tasks/impl/Stresstest.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/tasks/impl/Stresstest.java index f392cfee7..7fae30836 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/tasks/impl/Stresstest.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/tasks/impl/Stresstest.java @@ -36,7 +36,6 @@ public class Stresstest extends AbstractTask { private final List warmupWorkers = new ArrayList<>(); private final List> workerConfig; protected List workers = new LinkedList<>(); - protected String cacheFolder = "queryInstances"; private Double warmupTimeMS; private Double timeLimit; private Long noOfQueryMixes; @@ -94,7 +93,6 @@ private int createWorker(Map workerConfig, List workersT //if shorthand classname is used, exchange to full classname workerConfig.put("connection", this.con); workerConfig.put("taskID", this.taskID); - addOutputFolder(workerConfig); if (timeLimit != null) { workerConfig.put("timeLimit", timeLimit.intValue()); @@ -111,11 +109,6 @@ private int createWorker(Map workerConfig, List workersT return threads; } - private void addOutputFolder(Map workerConfig) { - Map queries = (Map) workerConfig.getOrDefault("queries", new HashMap<>()); - queries.put("outputFolder", this.cacheFolder); - } - public void generateTripleStats() { // TODO How are these Triplestats used? diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/AbstractWorker.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/AbstractWorker.java index 004f66797..cb6534ac7 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/AbstractWorker.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/AbstractWorker.java @@ -4,7 +4,6 @@ import org.aksw.iguana.cc.config.elements.Connection; import org.aksw.iguana.cc.model.QueryExecutionStats; import org.aksw.iguana.cc.query.handler.QueryHandler; -import org.aksw.iguana.cc.query.set.QuerySet; import org.aksw.iguana.commons.annotation.Nullable; import org.aksw.iguana.commons.annotation.Shorthand; import org.aksw.iguana.commons.constants.COMMON; @@ -85,10 +84,6 @@ public AbstractWorker(String taskID, Integer workerID, Connection connection, Ma LOGGER.debug("Initialized new Worker[{{}} : {{}}] for taskID {{}}", this.workerType, workerID, taskID); } - public AbstractWorker(String taskID, Connection connection, String queriesFile, @Nullable Integer timeOut, @Nullable Integer timeLimit, @Nullable Integer fixedLatency, @Nullable Integer gaussianLatency, Integer workerID) { - // FIXME use other constructor in child classes - } - @Override public void waitTimeMs() { @@ -247,25 +242,6 @@ public void run() { startWorker(); } - /** - * Returns the name of the queries file name/update path - * - * @return file name/update path - */ - public String getQueriesFileName() { - // FIXME remove - return ""; - } - - /** - * Sets the Query Instances repr. in Files. - * - * @param queries File containing the query instances. - */ - public void setQueriesList(QuerySet[] queries) { - // FIXME remove - } - /** * The number of Queries in one mix * diff --git a/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/impl/PatternBasedQueryHandlerTest.java b/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/impl/PatternBasedQueryHandlerTest.java deleted file mode 100644 index e7b1888bf..000000000 --- a/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/impl/PatternBasedQueryHandlerTest.java +++ /dev/null @@ -1,187 +0,0 @@ -package org.aksw.iguana.cc.query.impl; - -import org.aksw.iguana.cc.config.elements.Connection; -import org.aksw.iguana.cc.query.set.QuerySet; -import org.aksw.iguana.cc.query.set.impl.FileBasedQuerySet; -import org.aksw.iguana.cc.worker.Worker; -import org.aksw.iguana.cc.worker.impl.SPARQLWorker; -import org.aksw.iguana.cc.worker.impl.UPDATEWorker; -import org.apache.commons.io.FileUtils; -import org.apache.jena.ext.com.google.common.collect.Lists; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; - -import java.io.File; -import java.io.IOException; -import java.io.PrintWriter; -import java.util.*; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -@RunWith(Parameterized.class) -public class PatternBasedQueryHandlerTest { - - private final boolean isUpdate; - private String[] queryStr; - private String dir = UUID.randomUUID().toString(); - private File queriesFile; - - @Parameterized.Parameters - public static Collection data(){ - Collection testData = new ArrayList(); - testData.add(new Object[]{new String[]{"SELECT * {?s ?p ?o}"}, false}); - testData.add(new Object[]{new String[]{"SELECT * {?s ?p ?o}"}, false}); - testData.add(new Object[]{new String[]{"UPDATE * {?s ?p ?o}"}, true}); - - return testData; - } - - public PatternBasedQueryHandlerTest(String[] queryStr, boolean isUpdate){ - this.queryStr = queryStr; - this.isUpdate=isUpdate; - } - - @Before - public void createFolder() throws IOException { - //File f = new File(this.dir); - //f.mkdir(); - String queryFile = UUID.randomUUID().toString(); - File f = new File(queryFile); - f.createNewFile(); - try(PrintWriter pw = new PrintWriter(f)){ - for(String query : queryStr) { - pw.println(query); - } - } - //remove empty lines after printing them, so the expected asserts will correctly assume that the empty limes are ignored - List tmpList = Lists.newArrayList(queryStr); - Iterator it = tmpList.iterator(); - while(it.hasNext()){ - if(it.next().isEmpty()){ - it.remove(); - } - } - this.queryStr= tmpList.toArray(new String[]{}); - this.queriesFile = f; - f.deleteOnExit(); - } - - @After - public void removeFolder() throws IOException { - File f = new File(this.dir); - FileUtils.deleteDirectory(f); - } - - - - @Test - public void testQueryCreation() throws IOException { - //Get queries file - Connection con = new Connection(); - con.setName("a"); - con.setEndpoint("http://test.com"); - Worker worker = getWorker(con, 1, "1"); - PatternQueryHandler qh = new PatternQueryHandler(Lists.newArrayList(worker), con.getEndpoint()); - qh.setOutputFolder(this.dir); - Map map = qh.generate(); - //check if folder exist this.dir/hashCode/ with |queries| files - int hashcode = org.aksw.iguana.cc.utils.FileUtils.getHashcodeFromFileContent(this.queriesFile.getAbsolutePath()); - File f = new File(this.dir+File.separator+hashcode); - if(!isUpdate) { - assertTrue(f.isDirectory()); - int expectedNoOfFiles = queryStr.length; - assertEquals(expectedNoOfFiles, f.listFiles().length); - //iterate through all and check if correct - HashSet files = new HashSet(); - for(File queryFile : f.listFiles()){ - int id = Integer.parseInt(queryFile.getName().replace("sparql", "").replace("update", "")); - String actualQueryString =org.aksw.iguana.cc.utils.FileUtils.readLineAt(0, queryFile); - assertEquals(queryStr[id], actualQueryString); - files.add(queryFile.getAbsolutePath()); - } - for(QuerySet querySet : map.get(this.queriesFile.getAbsolutePath())){ - if(querySet instanceof FileBasedQuerySet) { - assertTrue(files.contains(((FileBasedQuerySet) querySet).getFile().getAbsolutePath())); - } } - assertEquals(files.size(), map.get(this.queriesFile.getAbsolutePath()).length); - FileUtils.deleteDirectory(f); - } - else{ - List expected = new ArrayList(); - List actual = new ArrayList(); - - for(String qStr : queryStr){ - expected.add(qStr); - } - - for(QuerySet querySet : map.get(this.queriesFile.getAbsolutePath())){ - assertEquals(1, querySet.size()); - actual.add(querySet.getQueryAtPos(0)); - } - assertEquals(expected.size(), actual.size()); - actual.removeAll(expected); - assertEquals(actual.size(),0); - assertEquals(queryStr.length, map.get(this.queriesFile.getAbsolutePath()).length); - } - - - } - - @Test - public void testCaching() throws IOException { - if(isUpdate){ - //nothing to check - return; - } - //Get queries file - Connection con = new Connection(); - con.setName("a"); - con.setEndpoint("http://test.com"); - Worker worker = getWorker(con, 1, "1"); - PatternQueryHandler qh = new PatternQueryHandler(Lists.newArrayList(worker), con.getEndpoint()); - qh.setOutputFolder(this.dir); - - Map queries1 = qh.generate(); - //check if folder exist this.dir/hashCode/ with |queries| files - int hashcode = org.aksw.iguana.cc.utils.FileUtils.getHashcodeFromFileContent(this.queriesFile.getAbsolutePath()); - File f = new File(this.dir+File.separator+hashcode); - - worker = getWorker(con, 12, "2"); - qh = new PatternQueryHandler(Lists.newArrayList(worker), con.getEndpoint()); - - qh.setOutputFolder(this.dir); - Map queries2 = qh.generate(); - - HashSet files = new HashSet(); - for(QuerySet querySet : queries1.get(this.queriesFile.getAbsolutePath())){ - if(querySet instanceof FileBasedQuerySet) { - files.add(((FileBasedQuerySet)querySet).getFile().getAbsolutePath()); - } - - } - for(QuerySet querySet : queries2.get(this.queriesFile.getAbsolutePath())){ - if(querySet instanceof FileBasedQuerySet) { - assertTrue(files.contains(((FileBasedQuerySet) querySet).getFile().getAbsolutePath())); - } - - } - - assertEquals(files.size(), queries2.get(this.queriesFile.getAbsolutePath()).length); - FileUtils.deleteDirectory(f); - } - - public Worker getWorker(Connection con, int id, String taskID){ - if(isUpdate){ - return new UPDATEWorker(taskID, con, this.queriesFile.getAbsolutePath(), null, null, null, null,null, id); - } - else { - return new SPARQLWorker(taskID, con, this.queriesFile.getAbsolutePath(), null, null, null, null, null, null, id); - } - } - - -} diff --git a/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/pattern/PatternBasedQueryHandlerTest.java b/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/pattern/PatternBasedQueryHandlerTest.java new file mode 100644 index 000000000..e6f264133 --- /dev/null +++ b/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/pattern/PatternBasedQueryHandlerTest.java @@ -0,0 +1,141 @@ +package org.aksw.iguana.cc.query.pattern; + +import org.aksw.iguana.cc.query.source.QuerySource; +import org.aksw.iguana.cc.query.source.impl.FileLineQuerySource; +import org.apache.commons.io.FileUtils; +import org.apache.jena.ext.com.google.common.collect.Lists; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import java.io.File; +import java.io.IOException; +import java.io.PrintWriter; +import java.nio.file.Files; +import java.util.*; +import java.util.stream.Stream; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +@RunWith(Parameterized.class) +public class PatternBasedQueryHandlerTest { + private final String dir = UUID.randomUUID().toString(); + private String[] queryStr; + private File queriesFile; + + public PatternBasedQueryHandlerTest(String[] queryStr) { + this.queryStr = queryStr; + } + + @Parameterized.Parameters + public static Collection data() { + Collection testData = new ArrayList<>(); + testData.add(new Object[]{new String[]{"SELECT * {?s ?p ?o}"}}); + testData.add(new Object[]{new String[]{"SELECT * {?s ?p ?o}"}}); + + return testData; + } + + @Before + public void createFolder() throws IOException { + //File f = new File(this.dir); + //f.mkdir(); + String queryFile = UUID.randomUUID().toString(); + File f = new File(queryFile); + f.createNewFile(); + try (PrintWriter pw = new PrintWriter(f)) { + for (String query : this.queryStr) { + pw.println(query); + } + } + //remove empty lines after printing them, so the expected asserts will correctly assume that the empty limes are ignored + List tmpList = Lists.newArrayList(this.queryStr); + Iterator it = tmpList.iterator(); + while (it.hasNext()) { + if (it.next().isEmpty()) { + it.remove(); + } + } + this.queryStr = tmpList.toArray(new String[]{}); + this.queriesFile = f; + f.deleteOnExit(); + } + + @After + public void removeFolder() throws IOException { + File f = new File(this.dir); + FileUtils.deleteDirectory(f); + } + + + @Test + public void testQueryCreation() throws IOException { + QuerySource originalSource = getQuerySource(); + PatternHandler ph = new PatternHandler(getConfig(), originalSource); + QuerySource qs = ph.generateQuerySource(); + + //check if folder exist this.dir/hashCode/ with |queries| files + int hashcode = originalSource.getHashcode(); + File f = new File(this.dir + File.separator + hashcode); + File outDir = new File(this.dir); + assertTrue(outDir.exists()); + assertTrue(outDir.isDirectory()); + assertTrue(f.isFile()); + + assertEquals(1, outDir.listFiles().length); + + int expectedNoOfQueries = this.queryStr.length; + assertEquals(expectedNoOfQueries, qs.size()); + + try (Stream stream = Files.lines(f.toPath())) { + assertEquals(expectedNoOfQueries, stream.count()); + } + + for (int i = 0; i < expectedNoOfQueries; i++) { + assertEquals(this.queryStr[i], qs.getQuery(i)); + } + + FileUtils.deleteDirectory(outDir); + } + + @Test + public void testCaching() throws IOException { + QuerySource originalSource = getQuerySource(); + PatternHandler ph = new PatternHandler(getConfig(), originalSource); + ph.generateQuerySource(); + + int hashcode = originalSource.getHashcode(); + File f = new File(this.dir + File.separator + hashcode); + assertTrue(f.exists()); + assertTrue(f.isFile()); + + int contentHash = org.aksw.iguana.cc.utils.FileUtils.getHashcodeFromFileContent(f.getAbsolutePath()); + Map attr = Files.readAttributes(f.toPath(), "basic:creationTime"); + + PatternHandler ph2 = new PatternHandler(getConfig(), originalSource); + ph2.generateQuerySource(); + + int contentHash2 = org.aksw.iguana.cc.utils.FileUtils.getHashcodeFromFileContent(f.getAbsolutePath()); + assertEquals(contentHash, contentHash2); + + Map attr2 = Files.readAttributes(f.toPath(), "basic:creationTime"); + assertEquals(attr.get("creationTime"), attr2.get("creationTime")); + + FileUtils.deleteDirectory(new File(this.dir)); + } + + private Map getConfig() { + Map config = new HashMap<>(); + config.put("endpoint", "http://test.com"); + config.put("outputFolder", this.dir); + config.put("limit", 5); + return config; + } + + private QuerySource getQuerySource() { + return new FileLineQuerySource(this.queriesFile.getAbsolutePath()); + } +} diff --git a/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/impl/PatternQueryHandlerTest.java b/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/pattern/PatternQueryHandlerTest.java similarity index 68% rename from iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/impl/PatternQueryHandlerTest.java rename to iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/pattern/PatternQueryHandlerTest.java index 41b4942e7..3117134ed 100644 --- a/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/impl/PatternQueryHandlerTest.java +++ b/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/pattern/PatternQueryHandlerTest.java @@ -1,11 +1,10 @@ -package org.aksw.iguana.cc.query.impl; +package org.aksw.iguana.cc.query.pattern; +import org.aksw.iguana.cc.query.source.QuerySource; +import org.aksw.iguana.cc.query.source.impl.FileLineQuerySource; +import org.aksw.iguana.cc.utils.ServerMock; import org.apache.jena.ext.com.google.common.collect.Lists; import org.apache.jena.ext.com.google.common.collect.Sets; -import org.aksw.iguana.cc.config.elements.Connection; -import org.aksw.iguana.cc.utils.ServerMock; -import org.aksw.iguana.cc.worker.Worker; -import org.aksw.iguana.cc.worker.impl.SPARQLWorker; import org.apache.jena.query.ParameterizedSparqlString; import org.apache.jena.query.Query; import org.apache.jena.query.QueryFactory; @@ -28,22 +27,30 @@ public class PatternQueryHandlerTest { private static final int FAST_SERVER_PORT = 8024; - private final String service; - private static ServerMock fastServerContainer; private static ContainerServer fastServer; private static SocketConnection fastConnection; - + private final String service; private final String queryStr; private final Query expectedConversionQuery; private final String[] vars; private final String expectedReplacedQuery; private final List expectedInstances; - private String dir = UUID.randomUUID().toString(); + private final String dir = UUID.randomUUID().toString(); + + + public PatternQueryHandlerTest(String queryStr, String expectedConversionStr, String expectedReplacedQuery, String[] vars, String[] expectedInstances) { + this.service = "http://localhost:8024"; + this.queryStr = queryStr; + this.expectedConversionQuery = QueryFactory.create(expectedConversionStr); + this.vars = vars; + this.expectedReplacedQuery = expectedReplacedQuery; + this.expectedInstances = Lists.newArrayList(expectedInstances); + } @Parameterized.Parameters - public static Collection data(){ - Collection testData = new ArrayList(); + public static Collection data() { + Collection testData = new ArrayList<>(); testData.add(new Object[]{"SELECT * {?s ?p ?o}", "SELECT * {?s ?p ?o}", "SELECT * {?s ?p ?o}", new String[]{}, new String[]{"SELECT * {?s ?p ?o}"}}); testData.add(new Object[]{"SELECT ?book {?book %%var0%% ?o}", "SELECT DISTINCT ?var0 {?book ?var0 ?o} LIMIT 2000", "SELECT ?book {?book ?var0 ?o}", new String[]{"var0"}, new String[]{"SELECT ?book {?book ?o}", "SELECT ?book {?book ?o}"}}); testData.add(new Object[]{"SELECT ?book {?book %%var0%% %%var1%%}", "SELECT DISTINCT ?var1 ?var0 {?book ?var0 ?var1} LIMIT 2000", "SELECT ?book {?book ?var0 ?var1}", new String[]{"var0", "var1"}, new String[]{"SELECT ?book {?book \"Example Book 2\"}", "SELECT ?book {?book \"Example Book 1\"}"}}); @@ -53,7 +60,7 @@ public static Collection data(){ @BeforeClass public static void startServer() throws IOException { - fastServerContainer = new ServerMock(); + ServerMock fastServerContainer = new ServerMock(); fastServer = new ContainerServer(fastServerContainer); fastConnection = new SocketConnection(fastServer); SocketAddress address1 = new InetSocketAddress(FAST_SERVER_PORT); @@ -66,54 +73,41 @@ public static void stopServer() throws IOException { fastServer.stop(); } - public PatternQueryHandlerTest(String queryStr, String expectedConversionStr, String expectedReplacedQuery, String[] vars, String[] expectedInstances) throws IOException { - this.service = "http://localhost:8024"; - - this.queryStr = queryStr; - this.expectedConversionQuery = QueryFactory.create(expectedConversionStr); - this.vars = vars; - this.expectedReplacedQuery=expectedReplacedQuery; - this.expectedInstances = Lists.newArrayList(expectedInstances); - } - @Test - public void testReplacement(){ - Set varNames = new HashSet(); + public void testReplacement() { + Set varNames = new HashSet<>(); String replacedQuery = getHandler().replaceVars(this.queryStr, varNames); - assertEquals(expectedReplacedQuery, replacedQuery); + assertEquals(this.expectedReplacedQuery, replacedQuery); assertEquals(Sets.newHashSet(vars), varNames); } @Test - public void testPatternExchange(){ - List instances = getHandler().getInstances(queryStr); - assertEquals(expectedInstances, instances); - + public void testPatternExchange() { + List instances = getHandler().generateQueries(this.queryStr); + assertEquals(this.expectedInstances, instances); } @Test - public void testConversion(){ + public void testConversion() { // convert query // retrieve instances - PatternQueryHandler qh = getHandler(); + PatternHandler qh = getHandler(); ParameterizedSparqlString pss = new ParameterizedSparqlString(); - pss.setCommandText(qh.replaceVars(queryStr, Sets.newHashSet())); + pss.setCommandText(qh.replaceVars(this.queryStr, Sets.newHashSet())); - Query q = qh.convertToSelect(pss, Sets.newHashSet(vars)); - assertEquals(expectedConversionQuery, q); + Query q = qh.convertToSelect(pss, Sets.newHashSet(this.vars)); + assertEquals(this.expectedConversionQuery, q); } - private PatternQueryHandler getHandler(){ - Connection con = new Connection(); - con.setName("a"); - con.setEndpoint("http://test.com"); - Worker worker = new SPARQLWorker("1", con, "empty.txt", null,null,null,null,null,null, 1); - - PatternQueryHandler qh = new PatternQueryHandler(Lists.newArrayList(worker), service); - return qh; - } + private PatternHandler getHandler() { + Map config = new HashMap<>(); + config.put("endpoint", this.service); + config.put("outputFolder", this.dir); + QuerySource qs = new FileLineQuerySource("src/test/resources/workers/single-query.txt"); + return new PatternHandler(config, qs); + } } diff --git a/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/tasks/impl/StresstestTest.java b/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/tasks/impl/StresstestTest.java index 35c02d6fd..f065b5dab 100644 --- a/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/tasks/impl/StresstestTest.java +++ b/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/tasks/impl/StresstestTest.java @@ -9,11 +9,8 @@ import org.aksw.iguana.rp.metrics.MetricManager; import org.aksw.iguana.rp.metrics.impl.EachQueryMetric; import org.aksw.iguana.rp.storage.StorageManager; -import org.apache.commons.io.FileUtils; import org.junit.Test; -import java.io.File; -import java.io.IOException; import java.time.Instant; import java.util.*; @@ -56,10 +53,9 @@ private void init(){ } @Test - public void checkStresstestNoQM() throws IOException { + public void checkStresstestNoQM() { Stresstest task = new Stresstest(getWorkers(2, this.queries), 10); - task.cacheFolder = UUID.randomUUID().toString(); task.init(new String[]{"1", "1/1", "1/1/1"}, "test", getConnection()); init(); @@ -68,14 +64,12 @@ public void checkStresstestNoQM() throws IOException { //2 queries in mix, 10 executions on 2 workers -> 40 queries assertEquals(40, task.getExecutedQueries()); - FileUtils.deleteDirectory(new File(task.cacheFolder)); } @Test - public void checkStresstestTL() throws IOException { + public void checkStresstestTL() { Stresstest task = new Stresstest(5000, getWorkers(2, this.queries)); - task.cacheFolder = UUID.randomUUID().toString(); task.init(new String[]{"1", "1/1", "1/1/1"}, "test", getConnection()); @@ -86,15 +80,13 @@ public void checkStresstestTL() throws IOException { Instant end = Instant.now(); //give about 200milliseconds time for init and end stuff assertEquals(5000.0, end.toEpochMilli() - start.toEpochMilli(), 300.0); - FileUtils.deleteDirectory(new File(task.cacheFolder)); } @Test - public void warmupTest() throws IOException { + public void warmupTest() { //check if not executing Stresstest task = new Stresstest(5000, getWorkers(2, this.queries)); - task.cacheFolder = UUID.randomUUID().toString(); task.init(new String[]{"1", "1/1", "1/1/1"}, "test", getConnection()); Instant start = Instant.now(); @@ -106,10 +98,8 @@ public void warmupTest() throws IOException { Map warmup = new LinkedHashMap<>(); warmup.put("workers", getWorkers(2, this.queries)); warmup.put("timeLimit", 350); - FileUtils.deleteDirectory(new File(task.cacheFolder)); task = new Stresstest(5000, getWorkers(2, this.queries), warmup); - task.cacheFolder = UUID.randomUUID().toString(); task.init(new String[]{"1", "1/1", "1/1/1"}, "test", getConnection()); start = Instant.now(); @@ -119,16 +109,14 @@ public void warmupTest() throws IOException { assertEquals(350.0, end.toEpochMilli() - start.toEpochMilli(), 250.0); //each worker could execute 3 query assertEquals(6, queriesExecuted); - FileUtils.deleteDirectory(new File(task.cacheFolder)); } @Test - public void workerCreationTest() throws IOException { + public void workerCreationTest() { List> worker = getWorkers(2, this.queries); worker.addAll(getWorkers(1, this.queries2)); Stresstest task = new Stresstest(5000, worker); - task.cacheFolder = UUID.randomUUID().toString(); task.init(new String[]{"1", "1/1", "1/1/1"}, "test", getConnection()); List workers = task.workers; @@ -147,7 +135,6 @@ public void workerCreationTest() throws IOException { } assertEquals(2, q1); assertEquals(1, q2); - FileUtils.deleteDirectory(new File(task.cacheFolder)); } } From c23cff3b37af8b92d8c2ae58cf387998bdaa27ae Mon Sep 17 00:00:00 2001 From: frensing Date: Fri, 4 Nov 2022 20:45:22 +0200 Subject: [PATCH 19/47] PatternHandler and remove of old QueryHandler --- .../java/org/aksw/iguana/cc/config/elements/Task.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/config/elements/Task.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/config/elements/Task.java index 6ba9b151e..781397f2c 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/config/elements/Task.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/config/elements/Task.java @@ -12,12 +12,12 @@ public class Task implements Serializable { @JsonProperty(required = true) - private Map configuration = new HashMap(); + private Map configuration = new HashMap<>(); @JsonProperty(required = true) private String className; - @JsonProperty(required = false) + @JsonProperty() private String name=null; public String getName() { @@ -28,11 +28,11 @@ public void setName(String name) { this.name = name; } - public Map getConfiguration() { + public Map getConfiguration() { return configuration; } - public void setConfiguration(Map configuration) { + public void setConfiguration(Map configuration) { this.configuration = configuration; } From 2228f4fcdb678bc65f18ac346d61cf86ffe4ca89 Mon Sep 17 00:00:00 2001 From: frensing Date: Fri, 4 Nov 2022 20:52:39 +0200 Subject: [PATCH 20/47] update QueryHandler to init pattern --- .../iguana/cc/query/handler/QueryHandler.java | 27 ++++++++++++++----- 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/handler/QueryHandler.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/handler/QueryHandler.java index 5fc13af10..ae705f67a 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/handler/QueryHandler.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/handler/QueryHandler.java @@ -2,6 +2,7 @@ import org.aksw.iguana.cc.lang.LanguageProcessor; import org.aksw.iguana.cc.lang.QueryWrapper; +import org.aksw.iguana.cc.query.pattern.PatternHandler; import org.aksw.iguana.cc.query.selector.QuerySelector; import org.aksw.iguana.cc.query.selector.impl.LinearQuerySelector; import org.aksw.iguana.cc.query.selector.impl.RandomQuerySelector; @@ -46,13 +47,16 @@ public QueryHandler(Map config, Integer workerID) { this.location = (String) config.get("location"); - initQuerySet(); + if (config.containsKey("pattern")) { + initPattern(); + } else { + initQuerySet(); + } + initQuerySelector(); initLanguageProcessor(); this.hashcode = this.querySet.getHashcode(); - - // TODO pattern } public void getNextQuery(StringBuilder queryStr, StringBuilder queryID) throws IOException { @@ -85,16 +89,27 @@ public LanguageProcessor getLanguageProcessor() { return this.langProcessor; } - private void initQuerySet() { + private void initPattern() { + Map patternConfig = (Map) this.config.get("pattern"); + PatternHandler patternHandler = new PatternHandler(patternConfig, createQuerySource()); + + initQuerySet(patternHandler.generateQuerySource()); + } + + private void initQuerySet(QuerySource querySource) { this.caching = (Boolean) this.config.getOrDefault("caching", true); if (this.caching) { - this.querySet = new InMemQuerySet(this.location, createQuerySource()); + this.querySet = new InMemQuerySet(this.location, querySource); } else { - this.querySet = new FileBasedQuerySet(this.location, createQuerySource()); + this.querySet = new FileBasedQuerySet(this.location, querySource); } } + private void initQuerySet() { + initQuerySet(createQuerySource()); + } + private QuerySource createQuerySource() { Object formatObj = this.config.getOrDefault("format", "one-per-line"); if (formatObj instanceof Map) { From 708a3f7ad30d9a990129b920e3eae232144432db Mon Sep 17 00:00:00 2001 From: frensing Date: Fri, 4 Nov 2022 21:10:27 +0200 Subject: [PATCH 21/47] add QueryHandlerTest with PatternHandler --- .../cc/query/handler/QueryHandlerTest.java | 46 +++++++++++++++++-- ...ndlerTest.java => PatternHandlerTest.java} | 4 +- .../test/resources/query/pattern-query.txt | 1 + 3 files changed, 45 insertions(+), 6 deletions(-) rename iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/pattern/{PatternQueryHandlerTest.java => PatternHandlerTest.java} (95%) create mode 100644 iguana.corecontroller/src/test/resources/query/pattern-query.txt diff --git a/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/handler/QueryHandlerTest.java b/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/handler/QueryHandlerTest.java index bd54473a0..b7c3e933a 100644 --- a/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/handler/QueryHandlerTest.java +++ b/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/handler/QueryHandlerTest.java @@ -1,23 +1,35 @@ package org.aksw.iguana.cc.query.handler; +import org.aksw.iguana.cc.utils.ServerMock; +import org.apache.commons.io.FileUtils; +import org.junit.AfterClass; +import org.junit.BeforeClass; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; +import org.simpleframework.http.core.ContainerServer; +import org.simpleframework.transport.connect.SocketConnection; +import java.io.File; import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.Map; +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.util.*; import static org.junit.Assert.assertEquals; @RunWith(Parameterized.class) public class QueryHandlerTest { + private static final int FAST_SERVER_PORT = 8024; + private static final String CACHE_FOLDER = UUID.randomUUID().toString(); + private static ContainerServer fastServer; + private static SocketConnection fastConnection; + private final QueryHandler queryHandler; private final String[] expected; + public QueryHandlerTest(Map config, String[] expected) { this.queryHandler = new QueryHandler(config, 0); // workerID 0 results in correct seed for RandomSelector this.expected = expected; @@ -66,9 +78,35 @@ public static Collection data() { config4.put("order", order4); testData.add(new Object[]{config4, random}); + String[] expectedInstances = new String[]{"SELECT ?book {?book ?o}", "SELECT ?book {?book ?o}", "SELECT ?book {?book ?o}", "SELECT ?book {?book ?o}"}; + Map config5 = new HashMap<>(); + config5.put("location", "src/test/resources/query/pattern-query.txt"); + Map pattern5 = new HashMap<>(); + pattern5.put("endpoint", "http://localhost:8024"); + pattern5.put("outputFolder", CACHE_FOLDER); + config5.put("pattern", pattern5); + testData.add(new Object[]{config5, expectedInstances}); + + return testData; } + @BeforeClass + public static void startServer() throws IOException { + ServerMock fastServerContainer = new ServerMock(); + fastServer = new ContainerServer(fastServerContainer); + fastConnection = new SocketConnection(fastServer); + SocketAddress address1 = new InetSocketAddress(FAST_SERVER_PORT); + fastConnection.connect(address1); + } + + @AfterClass + public static void stopServer() throws IOException { + fastConnection.close(); + fastServer.stop(); + FileUtils.deleteDirectory(new File(CACHE_FOLDER)); + } + @Test public void getNextQueryTest() throws IOException { StringBuilder query = new StringBuilder(); diff --git a/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/pattern/PatternQueryHandlerTest.java b/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/pattern/PatternHandlerTest.java similarity index 95% rename from iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/pattern/PatternQueryHandlerTest.java rename to iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/pattern/PatternHandlerTest.java index 3117134ed..c618815d9 100644 --- a/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/pattern/PatternQueryHandlerTest.java +++ b/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/pattern/PatternHandlerTest.java @@ -24,7 +24,7 @@ import static org.junit.Assert.assertEquals; @RunWith(Parameterized.class) -public class PatternQueryHandlerTest { +public class PatternHandlerTest { private static final int FAST_SERVER_PORT = 8024; private static ContainerServer fastServer; @@ -38,7 +38,7 @@ public class PatternQueryHandlerTest { private final String dir = UUID.randomUUID().toString(); - public PatternQueryHandlerTest(String queryStr, String expectedConversionStr, String expectedReplacedQuery, String[] vars, String[] expectedInstances) { + public PatternHandlerTest(String queryStr, String expectedConversionStr, String expectedReplacedQuery, String[] vars, String[] expectedInstances) { this.service = "http://localhost:8024"; this.queryStr = queryStr; diff --git a/iguana.corecontroller/src/test/resources/query/pattern-query.txt b/iguana.corecontroller/src/test/resources/query/pattern-query.txt new file mode 100644 index 000000000..612618757 --- /dev/null +++ b/iguana.corecontroller/src/test/resources/query/pattern-query.txt @@ -0,0 +1 @@ +SELECT ?book {?book %%var0%% ?o} \ No newline at end of file From 4f03a623ff7970ac09de1ec69f27902192b7756c Mon Sep 17 00:00:00 2001 From: frensing Date: Mon, 7 Nov 2022 12:00:58 +0200 Subject: [PATCH 22/47] add requested changes --- .../main/java/org/aksw/iguana/cc/tasks/impl/Stresstest.java | 2 +- .../src/main/java/org/aksw/iguana/cc/utils/FileUtils.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/tasks/impl/Stresstest.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/tasks/impl/Stresstest.java index 7fae30836..8dc352858 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/tasks/impl/Stresstest.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/tasks/impl/Stresstest.java @@ -306,7 +306,7 @@ protected boolean isFinished() { // UpdateWorker are allowed to execute all their updates boolean endFlag = true; for (Worker worker : this.workers) { - LOGGER.debug("No of query Mixes: {} , queriesInMix {}", worker.getExecutedQueries(), this.noOfQueryMixes); + LOGGER.debug("No of query Mixes: {} , queriesInMix {}", this.noOfQueryMixes, worker.getExecutedQueries()); //Check for each worker, if the if (worker.hasExecutedNoOfQueryMixes(this.noOfQueryMixes)) { if (!worker.isTerminated()) { diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/utils/FileUtils.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/utils/FileUtils.java index 3ba6269a6..0682d1f34 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/utils/FileUtils.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/utils/FileUtils.java @@ -14,8 +14,8 @@ public class FileUtils { /** - * Counts the lines in a file efficently Props goes to: - * ... + * Counts the lines in a file efficiently. Props goes to: + * http://stackoverflow.com/a/453067/2917596 * * @param filename File to count lines of * @return No. of lines in File From 39fa0006612c3da6716a757d7655fce0e986b2db Mon Sep 17 00:00:00 2001 From: frensing Date: Mon, 7 Nov 2022 14:15:37 +0200 Subject: [PATCH 23/47] rm tripleStats todo --- .../java/org/aksw/iguana/cc/tasks/impl/Stresstest.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/tasks/impl/Stresstest.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/tasks/impl/Stresstest.java index 8dc352858..b8ededdc9 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/tasks/impl/Stresstest.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/tasks/impl/Stresstest.java @@ -8,6 +8,7 @@ import org.aksw.iguana.commons.annotation.Shorthand; import org.aksw.iguana.commons.constants.COMMON; import org.apache.jena.rdf.model.Model; +import org.apache.jena.rdf.model.ModelFactory; import org.apache.jena.riot.RDFDataMgr; import org.apache.jena.riot.RDFFormat; import org.slf4j.Logger; @@ -110,15 +111,14 @@ private int createWorker(Map workerConfig, List workersT } public void generateTripleStats() { - // TODO How are these Triplestats used? - StringWriter sw = new StringWriter(); + Model tripleStats = ModelFactory.createDefaultModel(); for (Worker worker : this.workers) { - Model tripleStats = worker.getQueryHandler().getTripleStats(this.taskID); - RDFDataMgr.write(sw, tripleStats, RDFFormat.NTRIPLES); + tripleStats.add(worker.getQueryHandler().getTripleStats(this.taskID)); } + RDFDataMgr.write(sw, tripleStats, RDFFormat.NTRIPLES); this.metaData.put(COMMON.SIMPLE_TRIPLE_KEY, sw.toString()); - //this.metaData.put(COMMON.QUERY_STATS, tripleStats); + this.metaData.put(COMMON.QUERY_STATS, tripleStats); } /** From 3c1b9f027030cdcf00b62e11c3bf4e33cd4270ea Mon Sep 17 00:00:00 2001 From: frensing Date: Mon, 7 Nov 2022 15:03:48 +0200 Subject: [PATCH 24/47] rm unused method --- .../main/java/org/aksw/iguana/cc/config/elements/Task.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/config/elements/Task.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/config/elements/Task.java index 781397f2c..036c6457b 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/config/elements/Task.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/config/elements/Task.java @@ -32,10 +32,6 @@ public Map getConfiguration() { return configuration; } - public void setConfiguration(Map configuration) { - this.configuration = configuration; - } - public String getClassName() { return className; } From b145eb376c8855c8cbcc7cf93d4a7630a6fc035d Mon Sep 17 00:00:00 2001 From: frensing Date: Mon, 7 Nov 2022 15:10:49 +0200 Subject: [PATCH 25/47] override hashCode method --- .../iguana/cc/query/handler/QueryHandler.java | 5 ++-- .../cc/query/pattern/PatternHandler.java | 2 +- .../cc/query/selector/QuerySelector.java | 5 ++++ .../iguana/cc/query/set/AbstractQuerySet.java | 4 +-- .../aksw/iguana/cc/query/set/QuerySet.java | 18 ++++++++++++- .../cc/query/source/AbstractQuerySource.java | 2 +- .../iguana/cc/query/source/QuerySource.java | 26 ++++++++++++++++++- .../query/source/impl/FolderQuerySource.java | 2 +- .../aksw/iguana/cc/worker/AbstractWorker.java | 8 ++---- .../org/aksw/iguana/cc/worker/Worker.java | 14 ++++++---- .../pattern/PatternBasedQueryHandlerTest.java | 4 +-- .../source/impl/FileLineQuerySourceTest.java | 4 +-- .../impl/FileSeparatorQuerySourceTest.java | 2 +- .../source/impl/FolderQuerySourceTest.java | 2 +- 14 files changed, 72 insertions(+), 26 deletions(-) diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/handler/QueryHandler.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/handler/QueryHandler.java index ae705f67a..6ffda81df 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/handler/QueryHandler.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/handler/QueryHandler.java @@ -56,7 +56,7 @@ public QueryHandler(Map config, Integer workerID) { initQuerySelector(); initLanguageProcessor(); - this.hashcode = this.querySet.getHashcode(); + this.hashcode = this.querySet.hashCode(); } public void getNextQuery(StringBuilder queryStr, StringBuilder queryID) throws IOException { @@ -77,7 +77,8 @@ public Model getTripleStats(String taskID) { return this.langProcessor.generateTripleStats(queries, "" + this.hashcode, taskID); } - public int getHashcode() { + @Override + public int hashCode() { return this.hashcode; } diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/pattern/PatternHandler.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/pattern/PatternHandler.java index 371deb06a..76e2b74ca 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/pattern/PatternHandler.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/pattern/PatternHandler.java @@ -29,7 +29,7 @@ public PatternHandler(Map config, QuerySource querySource) { } public QuerySource generateQuerySource() { - File cacheFile = new File(this.outputFolder + File.separator + this.querySource.getHashcode()); + File cacheFile = new File(this.outputFolder + File.separator + this.querySource.hashCode()); if (cacheFile.exists()) { LOGGER.warn("Output folder already exists. Will not generate queries again. To generate them new remove the {{}} folder", cacheFile.getAbsolutePath()); diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/selector/QuerySelector.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/selector/QuerySelector.java index fb08a1700..27f139e4b 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/selector/QuerySelector.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/selector/QuerySelector.java @@ -1,5 +1,10 @@ package org.aksw.iguana.cc.query.selector; public interface QuerySelector { + /** + * This method gives the next query index that should be used. + * + * @return the next query index + */ int getNextIndex(); } diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/AbstractQuerySet.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/AbstractQuerySet.java index a03768656..3c81a80e9 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/AbstractQuerySet.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/AbstractQuerySet.java @@ -23,7 +23,7 @@ public String getName() { } @Override - public int getHashcode() { - return this.querySource.getHashcode(); + public int hashCode() { + return this.querySource.hashCode(); } } diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/QuerySet.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/QuerySet.java index 124f8aa40..c873635ce 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/QuerySet.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/QuerySet.java @@ -16,9 +16,25 @@ public interface QuerySet { */ String getQueryAtPos(int pos) throws IOException; + /** + * Gets no. of queries in the query set + * + * @return The no of queries in the query set + */ int size(); + /** + * Gets the name of the query set + * + * @return The name of the query set + */ String getName(); - int getHashcode(); + /** + * Gets the hashcode of the query set which is the hashcode of the query source + * + * @return The hashcode of the query set + */ + @Override + int hashCode(); } diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/AbstractQuerySource.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/AbstractQuerySource.java index 23d5bffb3..9b70065c5 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/AbstractQuerySource.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/AbstractQuerySource.java @@ -11,7 +11,7 @@ public AbstractQuerySource(String path) { } @Override - public int getHashcode() { + public int hashCode() { return FileUtils.getHashcodeFromFileContent(this.path); } } diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/QuerySource.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/QuerySource.java index 342bf7a55..8031b5fa6 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/QuerySource.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/QuerySource.java @@ -5,11 +5,35 @@ public interface QuerySource { + /** + * This method returns the number of queries in the source. + * + * @return the number of queries in the source + */ int size(); + /** + * This method returns the query at the given index. + * + * @param index the index of the query counted from the first query (in the first file) + * @return String of the query + * @throws IOException + */ String getQuery(int index) throws IOException; + /** + * This method returns all queries in the source as Strings. + * + * @return List of Strings of all queries + * @throws IOException + */ List getAllQueries() throws IOException; - int getHashcode(); + /** + * This method returns the hashcode of the source, calculated from the file contents rather than the Java object. + * + * @return the hashcode of the source + */ + @Override + int hashCode(); } diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/impl/FolderQuerySource.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/impl/FolderQuerySource.java index 1d95ef813..8581c4f94 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/impl/FolderQuerySource.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/impl/FolderQuerySource.java @@ -66,7 +66,7 @@ public List getAllQueries() throws IOException { } @Override - public int getHashcode() { + public int hashCode() { return FileUtils.getHashcodeFromFileContent(this.files[0].getAbsolutePath()); } } diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/AbstractWorker.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/AbstractWorker.java index cb6534ac7..9fa96cee4 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/AbstractWorker.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/AbstractWorker.java @@ -114,7 +114,7 @@ public void startWorker() { // For Update and Logging purpose get startTime of Worker this.startTime = Instant.now(); - this.queryHash = this.queryHandler.getHashcode(); + this.queryHash = this.queryHandler.hashCode(); LOGGER.info("Starting Worker[{{}} : {{}}].", this.workerType, this.workerID); // Execute Queries as long as the Stresstest will need. @@ -242,11 +242,7 @@ public void run() { startWorker(); } - /** - * The number of Queries in one mix - * - * @return - */ + @Override public long getNoOfQueries() { return this.queryHandler.getQueryCount(); } diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/Worker.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/Worker.java index 36ae5cd76..7580e911f 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/Worker.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/Worker.java @@ -37,7 +37,11 @@ public interface Worker extends Runnable{ */ void getNextQuery(StringBuilder queryStr, StringBuilder queryID) throws IOException; - + /** + * This method will return the query handler which is used by this worker + * + * @return QueryHandler which is used by this worker + */ QueryHandler getQueryHandler(); @@ -72,14 +76,14 @@ public interface Worker extends Runnable{ /** * Returns the no of queries in the queryset of the worker - * @return + * @return no of queries in the queryset */ long getNoOfQueries(); /** * Returns if the no of query mixes were already executed - * @param noOfQueryMixes - * @return + * @param noOfQueryMixes no of query mixes + * @return true if the no of query mixes were already executed */ boolean hasExecutedNoOfQueryMixes(Long noOfQueryMixes); @@ -87,7 +91,7 @@ public interface Worker extends Runnable{ /** * Sets the end restriction * - * @param noOfQueryMixes + * @param noOfQueryMixes after which the worker should stop */ void endAtNoOfQueryMixes(Long noOfQueryMixes); diff --git a/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/pattern/PatternBasedQueryHandlerTest.java b/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/pattern/PatternBasedQueryHandlerTest.java index e6f264133..2c082531c 100644 --- a/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/pattern/PatternBasedQueryHandlerTest.java +++ b/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/pattern/PatternBasedQueryHandlerTest.java @@ -78,7 +78,7 @@ public void testQueryCreation() throws IOException { QuerySource qs = ph.generateQuerySource(); //check if folder exist this.dir/hashCode/ with |queries| files - int hashcode = originalSource.getHashcode(); + int hashcode = originalSource.hashCode(); File f = new File(this.dir + File.separator + hashcode); File outDir = new File(this.dir); assertTrue(outDir.exists()); @@ -107,7 +107,7 @@ public void testCaching() throws IOException { PatternHandler ph = new PatternHandler(getConfig(), originalSource); ph.generateQuerySource(); - int hashcode = originalSource.getHashcode(); + int hashcode = originalSource.hashCode(); File f = new File(this.dir + File.separator + hashcode); assertTrue(f.exists()); assertTrue(f.isFile()); diff --git a/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/source/impl/FileLineQuerySourceTest.java b/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/source/impl/FileLineQuerySourceTest.java index d9a969c7c..7801fec80 100644 --- a/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/source/impl/FileLineQuerySourceTest.java +++ b/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/source/impl/FileLineQuerySourceTest.java @@ -15,7 +15,7 @@ public class FileLineQuerySourceTest { private final FileLineQuerySource querySource; - public FileLineQuerySourceTest() throws IOException { + public FileLineQuerySourceTest() { this.querySource = new FileLineQuerySource(PATH); } @@ -44,6 +44,6 @@ public void getAllQueriesTest() throws IOException { @Test public void getHashcodeTest() { int expected = FileUtils.getHashcodeFromFileContent(PATH); - assertEquals(expected, this.querySource.getHashcode()); + assertEquals(expected, this.querySource.hashCode()); } } \ No newline at end of file diff --git a/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/source/impl/FileSeparatorQuerySourceTest.java b/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/source/impl/FileSeparatorQuerySourceTest.java index b1c849330..48e6c6e04 100644 --- a/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/source/impl/FileSeparatorQuerySourceTest.java +++ b/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/source/impl/FileSeparatorQuerySourceTest.java @@ -63,6 +63,6 @@ public void getAllQueriesTest() throws IOException { @Test public void getHashcodeTest() { int expected = FileUtils.getHashcodeFromFileContent(this.path); - assertEquals(expected, this.querySource.getHashcode()); + assertEquals(expected, this.querySource.hashCode()); } } \ No newline at end of file diff --git a/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/source/impl/FolderQuerySourceTest.java b/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/source/impl/FolderQuerySourceTest.java index 6703e397f..66cc37680 100644 --- a/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/source/impl/FolderQuerySourceTest.java +++ b/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/source/impl/FolderQuerySourceTest.java @@ -44,6 +44,6 @@ public void getAllQueriesTest() throws IOException { @Test public void getHashcodeTest() { int expected = FileUtils.getHashcodeFromFileContent(PATH + "/query1.txt"); - assertEquals(expected, this.querySource.getHashcode()); + assertEquals(expected, this.querySource.hashCode()); } } \ No newline at end of file From 35a8d42b44b152551171bbe57b74138bb99ae579 Mon Sep 17 00:00:00 2001 From: frensing Date: Sun, 13 Nov 2022 21:53:02 +0200 Subject: [PATCH 26/47] documentation and version update --- README.md | 4 +- docs/develop/extend-task.md | 46 ++--- docs/develop/extend-workers.md | 68 +------ docs/quick-config.md | 13 +- docs/shorthand-mapping.md | 5 - docs/usage/configuration.md | 36 ++-- docs/usage/getting-started.md | 10 +- docs/usage/queries.md | 236 ++++++++++++++++------ docs/usage/results.md | 3 +- docs/usage/stresstest.md | 127 +++++++----- docs/usage/tutorial.md | 69 ++++--- docs/usage/workers.md | 357 ++++++++++++++++----------------- example-suite.yml | 19 +- pom.xml | 6 +- schema/iguana.owl | 6 +- 15 files changed, 535 insertions(+), 470 deletions(-) diff --git a/README.md b/README.md index 1b8c5f00f..a83d95f92 100644 --- a/README.md +++ b/README.md @@ -90,8 +90,8 @@ Please download the release zip **iguana-x.y.z.zip** from the newest release ava ``` mkdir iguana -wget https://github.com/dice-group/IGUANA/releases/download/v3.3.2/iguana-3.3.2.zip -unzip iguana-3.3.2.zip +wget https://github.com/dice-group/IGUANA/releases/download/v4.0.0/iguana-4.0.0.zip +unzip iguana-4.0.0.zip ``` diff --git a/docs/develop/extend-task.md b/docs/develop/extend-task.md index df83e57f4..6c49a8109 100644 --- a/docs/develop/extend-task.md +++ b/docs/develop/extend-task.md @@ -7,10 +7,10 @@ You will need to create your own task either in the Iguana code itself or by usi Either way start by extending the AbstractTask. ```java -package org.benchmark +package org.benchmark; @Shorthand("MyBenchmarkTask") -public class MyBenchmarkTask extend AbstractTask { +public class MyBenchmarkTask extends AbstractTask { } @@ -19,14 +19,14 @@ public class MyBenchmarkTask extend AbstractTask { You will need to override some functions. For now include them and go through them step by step ```java -package org.benchmark +package org.benchmark; @Shorthand("MyBenchmarkTask") -public class MyBenchmarkTask extend AbstractTask { +public class MyBenchmarkTask extends AbstractTask { - //Your constructor(s) - public MyBenchmarkTask(Integer timeLimit, ArrayList workers, LinkedHashMap queryHandler) throws FileNotFoundException { - } + //Your constructor(s) + public MyBenchmarkTask(Integer timeLimit, List workers, Map config) throws FileNotFoundException { + } //Meta Data (which will be added in the resultsfile) @@ -70,8 +70,8 @@ The third parameter should provide a map of specific key-value pairs. You can set this up by using the following parameters: ```java -public MyBenchmarkTask(Integer param1, ArrayList param2, LinkedHashMap param3) throws FileNotFoundException { - //TODO whatever you need to do with the parameters +public MyBenchmarkTask(Integer param1,List param2,Map param3)throws FileNotFoundException{ + //TODO whatever you need to do with the parameters } ``` @@ -172,26 +172,26 @@ Simply override the existing one and call the super method and implement what yo ## Full overview ```java -package org.benchmark +package org.benchmark; @Shorthand("MyBenchmarkTask") -public class MyBenchmarkTask extend AbstractTask { +public class MyBenchmarkTask extends AbstractTask { - private Integer param1; - private ArrayList param2; - private LinkedHashMap param3; + private Integer param1; + private List param2; + private Map param3; - //Your constructor(s) - public MyBenchmarkTask(Integer param1, ArrayList param2, LinkedHashMap param3) throws FileNotFoundException { - - this.param1=param1; - this.param2=param2; - this.param3=param3; - - } + //Your constructor(s) + public MyBenchmarkTask(Integer param1, List param2, Map param3) throws FileNotFoundException { + this.param1 = param1; + this.param2 = param2; + this.param3 = param3; - //Meta Data (which will be added in the resultsfile) + } + + + //Meta Data (which will be added in the resultsfile) @Override public void addMetaData() { super.addMetaData(); diff --git a/docs/develop/extend-workers.md b/docs/develop/extend-workers.md index c85f838aa..e900c1922 100644 --- a/docs/develop/extend-workers.md +++ b/docs/develop/extend-workers.md @@ -5,17 +5,10 @@ If the implemented workers aren't sufficient you can create your own one. Start by extending the `AbstractWorker` ```java -package org.benchmark.workers +package org.benchmark.workers; @Shorthand("MyWorker") public class MyWorker extends AbstractWorker{ - - - //Setting the next query to be benchmarked in queryStr and queryID - public void getNextQuery(StringBuilder queryStr, StringBuilder queryID) throws IOException{ - - } - //Executing the current benchmark query public void executeQuery(String query, String queryID){ @@ -31,65 +24,12 @@ You can override more functions, please consider looking into the javadoc for th ## Constructor -The constructor parameters will be provided the same way the Task get's the parameters, thus simply look at [Extend Task](../extend-task). - -## Get the next query - -The benchmark task should create and initialize the benchmark queries and will set them accordingly to the worker. - -You can access these queries using the `queryFileList` array. -Each element consists of one query set, containing the queryID/name and a list of one to several queries. - -In the following we will choose the next query set, counted by `currentQueryID` and use a random query of this. - -```java - - @Override - public void getNextQuery(StringBuilder queryStr, StringBuilder queryID) throws IOException { - // get next Query File and next random Query out of it. - QuerySet currentQuery = this.queryFileList[this.currentQueryID++]; - queryID.append(currentQuery.getName()); - - int queriesInSet = currentQuery.size(); - int queryLine = queryChooser.nextInt(queriesInSet); - queryStr.append(currentQuery.getQueryAtPos(queryLine)); - - // If there is no more query(Pattern) start from beginning. - if (this.currentQueryID >= this.queryFileList.length) { - this.currentQueryID = 0; - } - - } -``` - -Thats it. - -This exact method is implemented in the `AbstractRandomQueryChooserWorker` class and instead of extend the `AbstractWorker` class, you can also extend this and spare your time. -However if you need another way like only executing one query and if there are no mery queries to test end the worker you can do so: - -```java - - @Override - public void getNextQuery(StringBuilder queryStr, StringBuilder queryID) throws IOException { - // If there is no more query(Pattern) start from beginning. - if (this.currentQueryID >= this.queryFileList.length) { - this.stopSending(); - } - - - // get next Query File and the first Query out of it. - QuerySet currentQuery = this.queryFileList[this.currentQueryID++]; - queryID.append(currentQuery.getName()); - - int queriesInSet = currentQuery.size(); - queryStr.append(currentQuery.getQueryAtPos(0)); - - } -``` +The constructor parameters will be provided the same way the Task gets the parameters, thus simply look +at [Extend Task](../extend-task). ## Execute the current query -Now you can execute the query against the current connection (`this.con`). +You can execute a query against the current connection (`this.con`). As this is up to you how to do that, here is an example implementation for using HTTP Get. diff --git a/docs/quick-config.md b/docs/quick-config.md index b498b74df..f45f99976 100644 --- a/docs/quick-config.md +++ b/docs/quick-config.md @@ -38,17 +38,16 @@ tasks: configuration: # 10 minutes (time Limit is in ms) timeLimit: 600000 - # we are using plain text queries - queryHandler: - className: "InstancesQueryHandler" - + # create one SPARQL Worker (it's basically a HTTP get worker using the 'query' parameter # it uses the queries.txt file as benchmark queries workers: - threads: 1 - className: "SPARQLWorker" - queriesFile: "queries.txt" - + className: "HttpGetWorker" + queries: + location: "queries.txt" + format: "one-per-line" + # tell Iguana where to save your results to storages: - className: "NTFileStorage" diff --git a/docs/shorthand-mapping.md b/docs/shorthand-mapping.md index e54e3132b..b4e43b815 100644 --- a/docs/shorthand-mapping.md +++ b/docs/shorthand-mapping.md @@ -2,15 +2,10 @@ |----------|-------| | Stresstest | `org.aksw.iguana.cc.tasks.impl.Stresstest` | |----------|-------| -| InstancesQueryHandler | `org.aksw.iguana.cc.query.impl.InstancesQueryHandler` | -| DelimInstancesQueryHandler | `org.aksw.iguana.cc.query.impl.DelimInstancesQueryHandler` | -| PatternQueryHandler | `org.aksw.iguana.cc.query.impl.PatternQueryHandler` | -|----------|-------| | lang.RDF | `org.aksw.iguana.cc.lang.impl.RDFLanguageProcessor` | | lang.SPARQL | `org.aksw.iguana.cc.lang.impl.SPARQLLanguageProcessor` | | lang.SIMPLE | `org.aksw.iguana.cc.lang.impl.ThrowawayLanguageProcessor` | |----------|-------| -| SPARQLWorker | `org.aksw.iguana.cc.worker.impl.SPARQLWorker` | | UPDATEWorker | `org.aksw.iguana.cc.worker.impl.UPDATEWorker` | | HttpPostWorker | `org.aksw.iguana.cc.worker.impl.HttpPostWorker` | | HttpGetWorker | `org.aksw.iguana.cc.worker.impl.HttpGetWorker` | diff --git a/docs/usage/configuration.md b/docs/usage/configuration.md index 070cb20f2..8568a4263 100644 --- a/docs/usage/configuration.md +++ b/docs/usage/configuration.md @@ -93,28 +93,28 @@ tasks: configuration: #timeLimit is in ms timeLimit: 3600000 - queryHandler: - className: "InstancesQueryHandler" workers: - threads: 2 - className: "SPARQLWorker" - queriesFile: "queries.txt" + className: "HttpGetWorker" + queries: + location: "queries.txt" timeOut: 180000 - className: "Stresstest" configuration: noOfQueryMixes: 1 - queryHandler: - className: "InstancesQueryHandler" workers: - threads: 2 - className: "SPARQLWorker" - queriesFile: "queries.txt" + className: "HttpGetWorker" + queries: + location: "queries.txt" timeOut: 180000 ``` -We configured two Tasks, both Stresstests. The first one will be executed for one hour and uses simple text queries which can be executed right away. -Further on it uses 2 simulated SPARQLWorkers with the same configuration. -At this point it's recommend to check out the [Stresstest Configuration](../stresstest#Configuration) in detail for further configuration. +We configured two Tasks, both Stresstests. The first one will be executed for one hour and uses simple text queries +which can be executed right away. +Further on it uses 2 simulated HttpGetWorkers with the same configuration. +At this point it's recommend to check out the [Stresstest Configuration](../stresstest#Configuration) in detail for +further configuration. ### Storages @@ -269,22 +269,20 @@ tasks: configuration: #timeLimit is in ms timeLimit: 3600000 - queryHandler: - className: "InstancesQueryHandler" workers: - threads: 2 - className: "SPARQLWorker" - queriesFile: "queries.txt" + className: "HttpGetWorker" + queries: + location: "queries.txt" timeOut: 180000 - className: "Stresstest" configuration: noOfQueryMixes: 1 - queryHandler: - className: "InstancesQueryHandler" workers: - threads: 2 - className: "SPARQLWorker" - queriesFile: "queries.txt" + className: "HttpGetWorker" + queries: + location: "queries.txt" timeOut: 180000 preScriptHook: "/full/path/{{ '{{connection}}' }}/load-and-start.sh {{ '{{dataset.file}}' }}" diff --git a/docs/usage/getting-started.md b/docs/usage/getting-started.md index 21ce6045e..d281333b8 100644 --- a/docs/usage/getting-started.md +++ b/docs/usage/getting-started.md @@ -15,11 +15,11 @@ Iguana is capable of benchmarking and stresstesting the following applications ### What Benchmarks are possible -Every simulated User (named Worker in the following) gets a set of queries. +Every simulated User (named Worker in the following) gets a set of queries. These queries have to be saved in one file, whereas each query is one line. -Hence everything you can fit in one line (e.g a SPARQL query, a text question, an RDF document) can be used as a query and a set of these queries represent the benchmark. -Iguana will then let every Worker execute these queries against the endpoint. - +Hence, everything you can fit in one line (e.g a SPARQL query, a text question, an RDF document) can be used as a query +and a set of these queries represent the benchmark. +Iguana will then let every Worker execute these queries against the endpoint. ## Download @@ -44,6 +44,6 @@ Start Iguana with a benchmark suite (e.g the example-suite.yml) either using the or using java 11 if you want to give Iguana more RAM or in general set JVM options. ```bash -java -jar iguana-corecontroller-3.3.2.jar example-suite.yml +java -jar iguana-corecontroller-4.0.0.jar example-suite.yml ``` diff --git a/docs/usage/queries.md b/docs/usage/queries.md index 74b8edfef..65508aab0 100644 --- a/docs/usage/queries.md +++ b/docs/usage/queries.md @@ -1,115 +1,223 @@ # Supported Queries -There are currently two query types supported: - -* plain text queries -* SPARQL pattern queries - -## Plain Text Queries +The queries are configured for each worker using the `queries` parameter. +The following parameters can be set: -This can be anything: SPARQL, SQL, a whole book if you need to. -The only limitation is that it has to fit in one line per query. If that isn't possible use the [Multiple Line Plain Text Queries](#multiple-line-plain-text-queries). -Every query can be executed as is. +| parameter | optional | default | description | +|-----------|----------|----------------|--------------------------------------------------------------------------------------------------------------------------------------| +| location | no | | The location of the queries | +| format | yes | "one-per-line" | Format how the queries are stored in the file(s) (see [Query Format](#query-format)) | +| caching | yes | true | Indicating if the queries should be loaded into memory or read from file when needed (see [Caching](#caching)) | +| order | yes | "linear" | The order in which the queries are read from the source (see [Query Order](#query-order)) | +| pattern | yes | | The configuration to be used to generate [SPARQL Pattern Queries](#sparql-pattern-queries) | +| lang | yes | "lang.SPARQL" | The language the queries and response are in (e.g. SPARQL). Basically just creates some more statistics (see [Langauge](#language) ) | -This can be set using the following: +For example: ```yaml -... - queryHandler: - className: "InstancesQueryHandler" +workers: + - queries: + location: "path/to/queries" + format: "one-per-line" + order: "random" + ... ``` -## SPARQL Pattern Queries - -This only works for SPARQL Queries at the moment. -The idea came from the DBpedia SPARQL Benchmark paper from 2011 and 2012. - -Instead of SPARQL queries as they are, you can set variables, which will be exchanged with real data. -Hence Iguana can create thousands of queries using a SPARQL pattern query. - -A pattern query might look like the following: -```sparql -SELECT * {?s rdf:type %%var0%% ; %%var1%% %%var2%%. %%var2%% ?p ?o} -``` +There are currently two query types supported: -This query in itself cannot be send to a triple store, however we can exchange the variables using real data. -Thus we need a reference endpoint (ideally) containing the same data as the dataset which will be tested. +- plain text queries +- SPARQL pattern queries -This query will then be exchanged to -```sparql -SELECT ?var0 ?var1 ?var2 {?s rdf:type ?var0 ; ?var1 ?var2. ?var2 ?p ?o} LIMIT 2000 -``` +The default are plain text queries, but SPARQL pattern queries can be used by setting the `pattern` parameter as +described in [SPARQL Pattern Queries](#sparql-pattern-queries). -and be queried against the reference endpoint. +## Query Format -For each result (limited to 2000) a query instance will be created. +A query can be anything: SPARQL, SQL, a whole book if you need to. +The queries can be provided in different formats: -This will be done for every query in the benchmark queries. -All instances of these query patterns will be subsummed as if they were one query in the results. +- one file with: + - one query per line + - multi line queries, separated by a separator line +- a folder with query files; one query per file -This can be set using the following: +The format is configured using the `format` parameter. -```yaml -... - queryHandler: - className: "PatternQueryHandler" - endpoint: "http://your-reference-endpoint/sparql" -``` +### One Query per Line -or +The queries are stored in one file, with one query per line. +The configuration for this format is: ```yaml -... - queryHandler: - className: "PatternQueryHandler" - endpoint: "http://your-reference-endpoint/sparql" - limit: 4000 +queries: + location: "path/to/queries" + format: "one-per-line" ``` -## Multiple Line Plain Text Queries +### Multi Line Queries -Basically like Plain Text Queries. However allows queries which need more than one line. -You basically seperate queries using a delimiter line. +The queries are stored in one file. Each query can span multiple lines and queries are separated by a separator line. -Let's look at an example, where the delimiter line is simply an empty line (this is the default) +Let's look at an example, where the separator line is "###" (this is the default) ``` QUERY 1 { still query 1 } - +### QUERY 2 { still Query2 } ``` -however if you set the delim=`###` for example the file has to look like: +The configuration for this format is: + +```yaml +queries: + location: "path/to/queries" + format: "separator" +``` + +However, you can set the separator in the configuration. +For example if the separator is an empty line, the file can look like this: ``` QUERY 1 { still query 1 } -### + QUERY 2 { still Query2 } ``` -The delimiter query handler can be set as follows +The configuration for this format is: + +```yaml +queries: + location: "path/to/queries" + format: + separator: "" +``` + +### Folder with Query Files + +The (multi-line) queries are stored in a folder with one query per file. Here it is important that the `location` is set +to a folder and not a file. +The configuration for this format is: + +```yaml +queries: + location: "path/to/queries" + format: "folder" +``` + +## Caching + +If the `caching` parameter is set to `true`, the queries are loaded into memory when the worker is initialized. This is +the **default**. +If the `caching` parameter is set to `false`, the queries are read from file when needed. This is useful if the queries +are very large, and you don't want all of them to be in memory at the same time. +An example configuration is: ```yaml -... - queryHandler: - className: "DelimInstancesQueryHandler" +queries: + location: "path/to/queries" + caching: false ``` -or if you want to set the delimiter line +## Query Order + +The queries can be read in different orders. The order is configured using the `order` parameter. + +### Linear Order + +The queries are read in the order they are stored in the file(s). This is the **default**. +The explicit configuration is: + +```yaml +queries: + location: "path/to/queries" + order: "linear" +``` + +### Random Order + +The queries are read in a (pseudo) random order. +The if no explicit seed is given, the generated workerID is used as seed, to ensure that each worker starts at the same +query each time. + +The configuration is: + +```yaml +queries: + location: "path/to/queries" + order: "linear" +``` + +If you want to use a specific seed, you can set it in the configuration: + +```yaml +queries: + location: "path/to/queries" + order: + random: + seed: 12345 +``` + +## Language + +The language of the queries and responses can be configured using the `lang` parameter. +This is used for generating statistics about the queries and responses. +For more information about supported languages see [Supported Langauges](languages). + +## SPARQL Pattern Queries + +This only works for SPARQL Queries at the moment. +The idea came from the DBpedia SPARQL Benchmark paper from 2011 and 2012. + +Instead of SPARQL queries as they are, you can set variables, which will be exchanged with real data. +Hence, Iguana can create thousands of queries using a SPARQL pattern query. + +A pattern query might look like the following: + +```sparql +SELECT * {?s rdf:type %%var0%% ; %%var1%% %%var2%%. %%var2%% ?p ?o} +``` + +This query in itself cannot be sent to a triple store, however we can exchange the variables using real data. +Thus, we need a reference endpoint (ideally) containing the same data as the dataset which will be tested. + +This query will then be exchanged to + +```sparql +SELECT ?var0 ?var1 ?var2 {?s rdf:type ?var0 ; ?var1 ?var2. ?var2 ?p ?o} LIMIT 2000 +``` + +and be queried against the reference endpoint. + +For each result (limited to 2000) a query instance will be created. + +This will be done for every query in the benchmark queries. +All instances of these query patterns will be subsumed as if they were one query in the results. + +The following parameters can be set: + +| parameter | optional | default | description | +|--------------|----------|--------------|----------------------------------------------------------------------------| +| endpoint | no | | The SPARQL endpoint used for filling the variables | +| limit | yes | 2000 | The limit how many instances should be created per query pattern | +| outputFolder | yes | "queryCache" | The folder where the file containing the generated queries will be located | + +An example configuration is: ```yaml -... - queryHandler: - className: "DelimInstancesQueryHandler" - delim: "###" +queries: + pattern: + endpoint: "http://your-reference-endpoint/sparql" + limit: 2000 + outputFolder: "queryCache" ``` +If the `outputFolder` contains a fitting cache file, the queries will not be generated again. diff --git a/docs/usage/results.md b/docs/usage/results.md index 68faf0ae6..0b77ca3b9 100644 --- a/docs/usage/results.md +++ b/docs/usage/results.md @@ -139,7 +139,8 @@ This provides the qps value, if the SPARQL query has a filter and the actual que ## Ontology -The results ontology (description of what each property and class means) can be found [here](http://iguana-benchmark.eu/ontology/3.3.2/iguana.owl) +The results' ontology (description of what each property and class means) can be +found [here](http://iguana-benchmark.eu/ontology/4.0.0/iguana.owl) diff --git a/docs/usage/stresstest.md b/docs/usage/stresstest.md index ea4c6302b..07aa31dfd 100644 --- a/docs/usage/stresstest.md +++ b/docs/usage/stresstest.md @@ -1,13 +1,20 @@ # Stresstest -Iguanas implemented Stresstest benchmark task tries to emulate a real case scenario under which an endpoint or application is under high stress. -As in real life endpoints might get multiple simultaneous request within seconds, it is very important to verify that you application can handle this. - -The stresstest emulates users or applications which will bombard the endpoint using a set of queries for a specific amount of time or a specific amount of queries executed. -Each simulated user is called Worker in the following. -As you might want to test read and write performance or just want to emulate different user behaviour, the stresstest allows to configure several workers. -Every worker configuration can additionaly be started several times, hence if you want one configuration executed multiple times, you can simply tell Iguana to run this worker configuration the specified amount of time. -However to assure that the endpoint can't just cache the repsonse of the first request of a query, every worker starts at a pre determined random query, meaning that the single worker will always start at that query to assure fairness in benchmark comparisons, while every worker will start at a different query. +Iguanas implemented Stresstest benchmark task tries to emulate a real case scenario under which an endpoint or +application is under high stress. +As in real life endpoints might get multiple simultaneous request within seconds, it is very important to verify that +you application can handle this. + +The stresstest emulates users or applications which will bombard the endpoint using a set of queries for a specific +amount of time or a specific amount of queries executed. +Each simulated user is called Worker in the following. +As you might want to test read and write performance or just want to emulate different user behaviour, the stresstest +allows to configure several workers. +Every worker configuration can additionally be started several times, hence if you want one configuration executed +multiple times, you can simply tell Iguana to run this worker configuration the specified amount of time. +However, to assure that the endpoint can't just cache the response of the first request of a query, every worker starts +at a pre-determined random query, meaning that the single worker will always start at that query to assure fairness in +benchmark comparisons, while every worker will start at a different query. ## Configuration @@ -28,64 +35,83 @@ tasks: ... ``` -As an end restriction you can either use `timeLimit` which will stop the stresstest after the specified amount in ms or you can set `noOfQueryMixes` which stops every worker after they executed the amount of queries in the provided query set. +As an end restriction you can either use `timeLimit` which will stop the stresstest after the specified amount in ms or +you can set `noOfQueryMixes` which stops every worker after they executed the amount of queries in the provided query +set. -Additionaly to either `timeLimit` or `noOfQueryMixes` you can set the following parameters +Additionally, to either `timeLimit` or `noOfQueryMixes` you can set the following parameters -* queryHandler * workers * warmup (optional) -### Query Handling - -The queryHandler parameter let's the stresstest know what queries will be used. -Normally you will need the `InstancesQueryHandler` which will use plain text queries (could be SQL, SPARQL, a whole RDF document). The only restriction is that each query has to be in one line. - -You can set the query handler like the following: -```yaml -tasks: - - className: "Stresstest" - queryHandler: - className: "InstancesQueryHandler" - ... -``` - -To see which query handlers are supported see [Supported Queries](../queries/) - ### Workers (simulated Users) Further on you have to add which workers to use. As described above you can set different worker configurations. Let's look at an example: + ```yaml - className: "Stresstest" timeLimit: 600000 workers: - threads: 4 - className: "SPARQLWorker" - queriesFile: "/path/to/your/queries.txt" + className: "HttpGetWorker" + queries: + location: "/path/to/your/queries.txt" - threads: 16 - className: "SPARQLWorker" - queriesFile: "/other/queries.txt" + className: "HttpGetWorker" + queries: + location: "/other/queries.txt" fixedLatency: 5000 ``` -In this example we have two different worker configurations we want to use. The first want will create 4 `SPARQLWorker`s using queries at `/path/to/your/queries.txt` with any latencym thus every query will be executed immediatly after another. -The second worker configuration will execute 16 `SPARQLWorker`s using queries at `/other/queries.txt` using a fixed waiting time of `5000ms` between each query. -Hence every worker will execute their queries independently from each other but will wait 5s after each of their query execution before executing the next one. -This configuration may simulate that we have a few Users requesting your endpoint locally (e.g. some of your application relying on your database) and several users querying your endpoint from outside the network where we would have network latency and other interferences which we will try to simulate with 5s. +In this example we have two different worker configurations we want to use. The first want will create 4 `HttpGetWorker` +s using queries at `/path/to/your/queries.txt` without any latency thus every query will be executed immediately after +another. +The second worker configuration will execute 16 `HttpGetWorker`s using queries at `/other/queries.txt` using a fixed +waiting time of `5000ms` between each query. +Hence, every worker will execute their queries independently of each other but will wait 5s after each of their query +execution before executing the next one. +This configuration may simulate that we have a few Users requesting your endpoint locally (e.g. some of your application +relying on your database) and several users querying your endpoint from outside the network where we would have network +latency and other interferences which we will try to simulate with 5s. A full list of supported workers and their parameters can be found at [Supported Workers](../workers) -In this example our Stresstest would create 20 workers, which will simultaenously request the endpoint for 60000ms (10 minutes). - -### Warmup +In this example our Stresstest would create 20 workers, which will simultaneously request the endpoint for 60000ms (10 +minutes). -Additionaly to these you can optionally set a warmup, which will aim to let the system be benchmarked under a normal situation (Some times a database is faster when it was already running for a bit) -The configuration is similar to the stresstest itself you can set a `timeLimit` (however not a certain no of query executions), you can set different `workers`, and a `queryHandler` to use. -If you don't set the `queryHandler` parameter the warmup will simply use the `queryHandler` specified in the Stresstest itself. +#### Query Handling + +The `queries` parameter lets the worker know what queries will be used. +The default is to have a single text file with one query per line (could be SQL, SPARQL, a whole RDF document). + +You can set the query handling like the following: + +```yaml +workers: + - className: "HttpGetWorker" + queries: + location: "/path/to/your/queries.txt" + format: "one-per-line" + order: + random: + seed: 1234 + ... +``` + +To see further configurations of the query handling see [Supported Queries](../queries/) + +### Warmup + +Additionaly to these you can optionally set a warmup, which will aim to let the system be benchmarked under a normal +situation (Some times a database is faster when it was already running for a bit) +The configuration is similar to the stresstest itself you can set a `timeLimit` (however not a certain no of query +executions), you can set different `workers` to use. +Each worker needs its own `queries` configuration. You can set the Warmup as following: + ```yaml tasks: - className: "Stresstest" @@ -93,8 +119,6 @@ tasks: timeLimit: 600000 workers: ... - queryHandler: - ... ``` That's it. @@ -114,19 +138,20 @@ tasks: # workers are set the same way as in the configuration part workers: - threads: 1 - className: "SPARQLWorker" - queriesFile: "queries_warmup.txt" + className: "HttpGetWorker" + queries: + location: "queries_warmup.txt" timeOut: 180000 - queryHandler: - className: "InstancesQueryHandler" workers: - threads: 16 - className: "SPARQLWorker" - queriesFile: "queries_easy.txt" + className: "HttpGetWorker" + queries: + location: "queries_easy.txt" timeOut: 180000 - threads: 4 - className: "SPARQLWorker" - queriesFile: "queries_complex.txt" + className: "HttpGetWorker" + queries: + location: "queries_complex.txt" fixedLatency: 100 ``` diff --git a/docs/usage/tutorial.md b/docs/usage/tutorial.md index bd9fc0313..23d4f3cbe 100644 --- a/docs/usage/tutorial.md +++ b/docs/usage/tutorial.md @@ -4,7 +4,7 @@ In this tutorial we will go through one benchmark using two systems, two dataset We are using the following -* Iguana v3.0.2 +* Iguana v4.0.0 * Apache Jena Fuseki 3 * Blazegraph @@ -20,9 +20,10 @@ cd myBenchmark Now let's download all required systems and Iguana. Starting with Iguana + ```bash -wget https://github.com/dice-group/IGUANA/releases/download/v3.0.2/iguana-3.0.2.zip -unzip iguana-3.0.2.zip +wget https://github.com/dice-group/IGUANA/releases/download/v4.0.0/iguana-4.0.0.zip +unzip iguana-4.0.0.zip ``` Now we will download Blazegraph @@ -33,7 +34,7 @@ wget https://downloads.sourceforge.net/project/bigdata/bigdata/2.1.5/blazegraph. cd ../ ``` -At last we just need to download Apache Jena Fuseki and Apache Jena +At last, we just need to download Apache Jena Fuseki and Apache Jena ```bash mkdir fuseki && cd fuseki @@ -44,8 +45,8 @@ wget https://downloads.apache.org/jena/binaries/apache-jena-fuseki-3.16.0.zip unzip apache-jena-fuseki-3.16.0.zip ``` -Finally we have to download our datasets. -We use two small datasets from scholarly data. +Finally, we have to download our datasets. +We use two small datasets from scholarly data. The ISWC 2010 and the ekaw 2012 rich dataset. ``` @@ -56,13 +57,13 @@ wget http://www.scholarlydata.org/dumps/conferences/alignments/ekaw-2012-complet cd .. ``` - That's it. -Let's setup blazegraph and fuseki. +Lets setup blazegraph and fuseki. ## Setting Up Systems -To simplify the benchmark workflow we will use the pre and post script hook, in which we will load the current system and after the benchmark stop the system. +To simplify the benchmark workflow we will use the pre- and post-script hook, in which we will load the current system +and after the benchmark stop the system. ### Blazegraph @@ -74,9 +75,10 @@ touch load-and-start.sh touch stop.sh ``` -The `load-and-start.sh` script will start blazegraph and use curl to POST our dataset. -In our case the datasets are pretty small, hence the loading time is minimal. -Otherwise it would be wise to load the dataset beforehand, backup the `blazegraph.jnl` file and simply exchanging the file in the pre script hook. +The `load-and-start.sh` script will start blazegraph and use curl to POST our dataset. +In our case the datasets are pretty small, hence the loading time is minimal. +Otherwise, it would be wise to load the dataset beforehand, backup the `blazegraph.jnl` file and simply exchanging the +file in the pre script hook. For now put this into the script `load-and-start.sh` @@ -152,9 +154,9 @@ cd .. ## Benchmark queries -We need some queries to benchmark. +We need some queries to benchmark. -For now we will just use 3 simple queryies +For now, we will just use 3 simple queries ``` SELECT * {?s ?p ?o} SELECT * {?s ?p ?o} LIMIT 10 @@ -174,9 +176,11 @@ Create a file called `benchmark-suite.yml` touch benchmark-suite.yml ``` -Add the following subscections to this file, or simply go to [#Full Configuration](full-configuration) and add the whole piece to it. +Add the following subsections to this file, or simply go to [#Full Configuration](configuration#full-example) and add +the whole piece to it. -Be aware that the configuration will be started on directory level below our working directory and thus paths will use `../` to get the correct path. +Be aware that the configuration will be started on directory level below our working directory and thus paths will +use `../` to get the correct path. ### Datasets @@ -221,23 +225,23 @@ post-script-hook: "../{{ '{{connection}}' }}/stop.sh ### Task configuration -We want to stresstest our stores using 10 minutes (60.000 ms)for each dataset connection pair. -We are using plain text queries (`InstancesQueryHandler`) and want to have two simulated users querying SPARQL queries. -The queries file is located at our working directory at `queries.txt`. Be aware that we start Iguana one level below, which makes the correct path `../queries.txt` +We want to stresstest our stores using 10 minutes (60.000 ms)for each dataset connection pair. +We are using plain text queries (one per line) and want to have two simulated users querying SPARQL queries. +The queries file is located at our working directory at `queries.txt`. Be aware that we start Iguana one level below, +which makes the correct path `../queries.txt` -To achieve this restrictions add the following to your file +To achieve these restrictions add the following to your file ```yaml tasks: - className: "Stresstest" configuration: timeLimit: 600000 - queryHandler: - className: "InstancesQueryHandler" workers: - threads: 2 - className: "SPARQLWorker" - queriesFile: "../queries.txt" + className: "HttpGetWorker" + queries: + location: "../queries.txt" ``` ### Result Storage @@ -267,20 +271,19 @@ connections: endpoint: "http://localhost:9999/blazegraph/sparql" - name: "fuseki" endpoint: "http://localhost:3030/ds/sparql" - + pre-script-hook: "../{{ '{{connection}}' }}/load-and-start.sh {{ '{{dataset.file}}' }}" post-script-hook: "../{{ '{{connection}}' }}/stop.sh tasks: - className: "Stresstest" - configuration: - timeLimit: 600000 - queryHandler: - className: "InstancesQueryHandler" - workers: - - threads: 2 - className: "SPARQLWorker" - queriesFile: "../queries.txt" + configuration: + timeLimit: 600000 + workers: + - threads: 2 + className: "HttpGetWorker" + queries: + location: "../queries.txt" storages: - className: "NTFileStorage" diff --git a/docs/usage/workers.md b/docs/usage/workers.md index a54794906..e1cec29ab 100644 --- a/docs/usage/workers.md +++ b/docs/usage/workers.md @@ -1,136 +1,138 @@ # Supported Workers -A Worker is basically just a thread querying the endpoint/application. It tries to emulate a single user/application requesting your system until it should stop. -In a task (e.g. the [stresstest](../stresstest/)) you can configure several worker configurations which will then be used inside the task. +A Worker is basically just a thread querying the endpoint/application. It tries to emulate a single user/application +requesting your system until it should stop. +In a task (e.g. the [stresstest](../stresstest/)) you can configure several worker configurations which will then be +used inside the task. -Every worker configuration can additionaly be started several times, hence if you want one configuration executed multiple times, you can simply tell Iguana to run this worker configuration the specified amount of time. -However to assure that the endpoint can't just cache the repsonse of the first request of a query, every worker starts at a pre determined random query, meaning that the single worker will always start at that query to assure fairness in benchmark comparisons, while every worker will start at a different query. +Every worker configuration can additionally be started several times, hence if you want one configuration executed +multiple times, you can simply tell Iguana to run this worker configuration the specified amount of time. +However, to assure that the endpoint can't just cache the response of the first request of a query, every worker starts +at a pre-determined random query, meaning that the single worker will always start at that query to assure fairness in +benchmark comparisons, while every worker will start at a different query. -There a few workers implemented, which can be seperated into two main categories +There are a few workers implemented, which can be seperated into two main categories * Http Workers * CLI Workers +## Common Configuration + +Every worker has the following configuration parameters: + +| parameter | optional | default | description | +|-----------------|----------|--------------------------|--------------------------------------------------------------------------------------------------------------------------------| +| threads | no | | The amount of workers to start using this worker configuration | +| queries | no | | Configuration for the queries this worker should use. (see [SupportedQueries](../queries)) | +| timeOut | yes | 180000 (3 minutes) | The timeout in MS after a query should be aborted | +| fixedLatency | yes | 0 | If the value (in MS) should be waited between each query. Simulating network latency or user behaviour. | +| gaussianLatency | yes | 0 | A random value between `[0, 2*value]` (in MS) will be waited between each query. Simulating network latency or user behaviour. | + ## Http Workers These Workers can be used to benchmark Http Applications (such as a SPARQL endpoint). ### Http Get Worker -A Http worker using GET requests. +A Http worker using GET requests. This worker will use the `endpoint` of the connection. This worker has several configurations listed in the following table: -| parameter | optional | default | description | -| ----- | --- | ----- | --------- | -| queriesFile | no | | File containg the queries this worker should use. | -| parameterName | yes | query | the GET paremter to set the query as value to. (see also [Supported Queries](../queries) ) | -| responseType | yes | | The content type the endpoint should return. Setting the `Accept: ` header | -| language | yes | lang.SPARQL (plain text) | The language the queries and response are in (e.g. SPARQL). Basically just creates some more statistics (see [Supported Langauges](languages) ) | -| timeOut | yes | 180000 (3 minutes) | The timeout in MS after a query should be aborted | -| fixedLatency | yes | 0 | If the value (in MS) should be waited between each query. Simulating network latency or user behaviour. | -| gaussianLatency | yes | 0 | A random value between `[0, 2*value]` (in MS) will be waited between each query. Simulating network latency or user behaviour. | +| parameter | optional | default | description | +|-----------------|----------|--------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------| +| parameterName | yes | query | The GET parameter to set the query as value to. (see also [Supported Queries](../queries) ) | +| responseType | yes | | The content type the endpoint should return. Setting the `Accept: ` header | Let's look at an example: ```yaml - ... - workers: - - threads: 1 - className: "HttpGetWorker" - timeOut: 180000 - parameterName: "text" + ... + workers: + - threads: 1 + className: "HttpGetWorker" + queries: + ... + timeOut: 180000 + parameterName: "text" ``` -This will use one HttpGetWOrker using a timeout of 3 minutes and the get parameter text to request the query through. +This will use one HttpGetWorker using a timeout of 3 minutes and the get parameter `text` to request the query through. ### Http Post Worker -A Http worker using POST requests. +A Http worker using POST requests. This worker will use the `updateEndpoint` of the connection. This worker has several configurations listed in the following table: -| parameter | optional | default | description | -| ----- | --- | ----- | --------- | -| queriesFile | no | | File containg the queries this worker should use. | -| parameterName | yes | query | the GET paremter to set the query as value to. (see also [Supported Queries](../queries) ) | -| contentType | yes | `text/plain` | The content type of the update queries. Setting the `Content-Type: ` header | -| responseType | yes | | The content type the endpoint should return. Setting the `Accept: ` header | -| language | yes | lang.SPARQL (plain text) | The language the queries and response are in (e.g. SPARQL). Basically just creates some more statistics (see [Supported Langauges](languages) ) | -| timeOut | yes | 180000 (3 minutes) | The timeout in MS after a query should be aborted | -| fixedLatency | yes | 0 | If the value (in MS) should be waited between each query. Simulating network latency or user behaviour. | -| gaussianLatency | yes | 0 | A random value between `[0, 2*value]` (in MS) will be waited between each query. Simulating network latency or user behaviour. | +| parameter | optional | default | description | +|-----------------|----------|--------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------| +| parameterName | yes | query | the GET parameter to set the query as value to. (see also [Supported Queries](../queries) ) | +| contentType | yes | `text/plain` | The content type of the update queries. Setting the `Content-Type: ` header | +| responseType | yes | | The content type the endpoint should return. Setting the `Accept: ` header | Let's look at an example: ```yaml - ... - workers: - - threads: 1 - className: "HttpPostWorker" - timeOut: 180000 + ... + workers: + - threads: 1 + className: "HttpPostWorker" + queries: + ... + timeOut: 180000 ``` -This will use one HttpGetWOrker using a timeout of 3 minutes. - -### SPARQL Worker +This will use one HttpPostWorker using a timeout of 3 minutes. -Simply a GET worker but the language parameter is set to `lang.SPARQL`. -Otherwise see the [Http Get Worker](#http-get-worker) for configuration - -An Example: -```yaml - ... - workers: - - threads: 1 - className: "SPARQLWorker" - timeOut: 180000 -``` +### SPARQL Worker - removed +Through the update of the query handling the `SPARQLWorker` is no longer different to the `HttpGetWorker`, since the +language parameter is set in the `queries` config rather than in the worker config. +Therefore we removed the SPARQL worker. ### SPARQL UPDATE Worker Simply a POST worker but specified for SPARQL Updates. -Parameters are : +Parameters are : -| parameter | optional | default | description | -| ----- | --- | ----- | --------- | -| queriesFile | no | | File containg the queries this worker should use. | -| timerStrategy | yes | `NONE` | `NONE`, `FIXED` or `DISTRIBUTED`. see below for explanation. | -| timeOut | yes | 180000 (3 minutes) | The timeout in MS after a query should be aborted | -| fixedLatency | yes | 0 | If the value (in MS) should be waited between each query. Simulating network latency or user behaviour. | -| gaussianLatency | yes | 0 | A random value between `[0, 2*value]` (in MS) will be waited between each query. Simulating network latency or user behaviour. | +| parameter | optional | default | description | +|-----------------|----------|--------------------|--------------------------------------------------------------------------------------------------------------------------------| +| timerStrategy | yes | `NONE` | `NONE`, `FIXED` or `DISTRIBUTED`. see below for explanation. | +The **timerStrategy** parameter lets the worker know how to distribute the updates. +The fixedLatency and gaussianLatency parameters are not affected, the worker will wait those additionally. -The **timerStrategy** parameter let's the worker know how to distribute the updates. -The fixedLatency and gaussianLatency parameters are not affected, the worker will wait those additionally. - -* NONE: the worker just updates each update query after another -* FIXED: calculating the distribution by `timeLimit / #updates` at the start and waiting the amount between each update. Time Limit will be used of the task the worker is executed in. -* DISTRIBUTED: calculating the time to wait between two updates after each update by `timeRemaining / #updatesRemaining`. +* NONE: the worker just updates each update query after another +* FIXED: calculating the distribution by `timeLimit / #updates` at the start and waiting the amount between each update. + Time Limit will be used of the task the worker is executed in. +* DISTRIBUTED: calculating the time to wait between two updates after each update + by `timeRemaining / #updatesRemaining`. An Example: + ```yaml - ... - workers: - - threads: 1 - className: "UPDATEWorker" - timeOut: 180000 - timerStrategy: "FIXED" + ... + workers: + - threads: 1 + className: "UPDATEWorker" + queries: + ... + timeOut: 180000 + timerStrategy: "FIXED" ``` - ## CLI Workers -These workers can be used to benchmark a CLI application. +These workers can be used to benchmark a CLI application. ### CLI Worker -This Worker should be used if the CLI application runs a query once and exits afterwards. -Something like +This Worker should be used if the CLI application runs a query once and exits afterwards. +Something like + ```bash $ cli-script.sh query HEADER @@ -140,30 +142,25 @@ QUERY RESULT 2 $ ``` -Parameters are : - -| parameter | optional | default | description | -| ----- | --- | ----- | --------- | -| queriesFile | no | | File containg the queries this worker should use. | -| timeOut | yes | 180000 (3 minutes) | The timeout in MS after a query should be aborted | -| fixedLatency | yes | 0 | If the value (in MS) should be waited between each query. Simulating network latency or user behaviour. | -| gaussianLatency | yes | 0 | A random value between `[0, 2*value]` (in MS) will be waited between each query. Simulating network latency or user behaviour. | - +This worker has no special parameters other than the [common parameters](#common-configuration). An Example: + ```yaml - ... - workers: - - threads: 1 - className: "CLIWorker" + ... + workers: + - threads: 1 + className: "CLIWorker" + queries: + ... ``` - ### CLI Input Worker -This Worker should be used if the CLI application runs and the query will be send using the Input. +This Worker should be used if the CLI application runs and the query will be sent using the Input. + +Something like -Something like ```bash $ cli-script.sh start Your Input: QUERY @@ -175,35 +172,35 @@ QUERY RESULT 2 Your Input: ``` -Parameters are : +Parameters are : -| parameter | optional | default | description | -| ----- | --- | ----- | --------- | -| queriesFile | no | | File containg the queries this worker should use. | -| initFinished | no | | String which occurs when the application is ready to be requested (e.g. after loading) | -| queryFinished | no | | String which occurs if the query response finished | -| queryError | no | | String which occurs when an error during the query execution happend | -| timeOut | yes | 180000 (3 minutes) | The timeout in MS after a query should be aborted | -| fixedLatency | yes | 0 | If the value (in MS) should be waited between each query. Simulating network latency or user behaviour. | -| gaussianLatency | yes | 0 | A random value between `[0, 2*value]` (in MS) will be waited between each query. Simulating network latency or user behaviour. | +| parameter | optional | default | description | +|-----------------|----------|--------------------|--------------------------------------------------------------------------------------------------------------------------------| +| initFinished | no | | String which occurs when the application is ready to be requested (e.g. after loading) | +| queryFinished | no | | String which occurs if the query response finished | +| queryError | no | | String which occurs when an error during the query execution happend | An Example: + ```yaml - ... - workers: - - threads: 1 - className: "CLIInputWorker" - initFinished: "loading finished" - queryFinished: "query execution took:" - queryError: "Error happend during request" + ... + workers: + - threads: 1 + className: "CLIInputWorker" + queries: + ... + initFinished: "loading finished" + queryFinished: "query execution took:" + queryError: "Error happened during request" ``` ### Multiple CLI Input Worker +This Worker should be used if the CLI application runs and the query will be sent using the Input and will quit on +errors. -This Worker should be used if the CLI application runs and the query will be send using the Input and will quit on errors. +Something like -Something like ```bash $ cli-script.sh start Your Input: QUERY @@ -217,38 +214,40 @@ ERROR happend, exiting $ ``` -To assure a smooth benchmark, the CLI application will be run multiple times instead of once, and if the application quits, the next running process will be used, while in the background the old process will be restarted. -Thus as soon as an error happend, the benchmark can continue without a problem. +To assure a smooth benchmark, the CLI application will be run multiple times instead of once, and if the application +quits, the next running process will be used, while in the background the old process will be restarted. +Thus, as soon as an error happened, the benchmark can continue without a problem. -Parameters are : +Parameters are : -| parameter | optional | default | description | -| ----- | --- | ----- | --------- | -| queriesFile | no | | File containg the queries this worker should use. | -| initFinished | no | | String which occurs when the application is ready to be requested (e.g. after loading) | -| queryFinished | no | | String which occurs if the query response finished | -| queryError | no | | String which occurs when an error during the query execution happend | -| numberOfProcesses | yes | 5 | The number of times the application should be started to assure a smooth benchmark. see above. | -| timeOut | yes | 180000 (3 minutes) | The timeout in MS after a query should be aborted | -| fixedLatency | yes | 0 | If the value (in MS) should be waited between each query. Simulating network latency or user behaviour. | -| gaussianLatency | yes | 0 | A random value between `[0, 2*value]` (in MS) will be waited between each query. Simulating network latency or user behaviour. | +| parameter | optional | default | description | +|-------------------|----------|--------------------|--------------------------------------------------------------------------------------------------------------------------------| +| initFinished | no | | String which occurs when the application is ready to be requested (e.g. after loading) | +| queryFinished | no | | String which occurs if the query response finished | +| queryError | no | | String which occurs when an error during the query execution happend | +| numberOfProcesses | yes | 5 | The number of times the application should be started to assure a smooth benchmark. see above. | An Example: + ```yaml - ... - workers: - - threads: 1 - className: "MultipleCLIInputWorker" - initFinished: "loading finished" - queryFinished: "query execution took:" - queryError: "Error happend during request" + ... + workers: + - threads: 1 + className: "MultipleCLIInputWorker" + queries: + ... + initFinished: "loading finished" + queryFinished: "query execution took:" + queryError: "Error happened during request" ``` ### CLI Input File Worker -Same as the [Multiple CLI Input Worker](#multiple-cli-input-worker). However the query won't be send to the input but written to a file and the file will be send to the input +Same as the [Multiple CLI Input Worker](#multiple-cli-input-worker). However, the query won't be sent to the input but +written to a file and the file will be sent to the input + +Something like -Something like ```bash $ cli-script.sh start Your Input: file-containg-the-query.txt @@ -259,38 +258,38 @@ QUERY RESULT 2 ``` -Parameters are : - -| parameter | optional | default | description | -| ----- | --- | ----- | --------- | -| queriesFile | no | | File containg the queries this worker should use. | -| initFinished | no | | String which occurs when the application is ready to be requested (e.g. after loading) | -| queryFinished | no | | String which occurs if the query response finished | -| queryError | no | | String which occurs when an error during the query execution happend | -| directory | no | | Directory in which the file including the query should be saved. | -| numberOfProcesses | yes | 5 | The number of times the application should be started to assure a smooth benchmark. see [Multiple CLI Input Worker](#multiple-cli-input-worker). | -| timeOut | yes | 180000 (3 minutes) | The timeout in MS after a query should be aborted | -| fixedLatency | yes | 0 | If the value (in MS) should be waited between each query. Simulating network latency or user behaviour. | -| gaussianLatency | yes | 0 | A random value between `[0, 2*value]` (in MS) will be waited between each query. Simulating network latency or user behaviour. | +Parameters are : +| parameter | optional | default | description | +|-------------------|----------|--------------------|--------------------------------------------------------------------------------------------------------------------------------------------------| +| initFinished | no | | String which occurs when the application is ready to be requested (e.g. after loading) | +| queryFinished | no | | String which occurs if the query response finished | +| queryError | no | | String which occurs when an error during the query execution happend | +| directory | no | | Directory in which the file including the query should be saved. | +| numberOfProcesses | yes | 5 | The number of times the application should be started to assure a smooth benchmark. see [Multiple CLI Input Worker](#multiple-cli-input-worker). | An Example: + ```yaml - ... - workers: - - threads: 1 - className: "CLIInputFileWorker" - initFinished: "loading finished" - queryFinished: "query execution took:" - queryError: "Error happend during request" - directory: "/tmp/" + ... + workers: + - threads: 1 + className: "CLIInputFileWorker" + queries: + ... + initFinished: "loading finished" + queryFinished: "query execution took:" + queryError: "Error happened during request" + directory: "/tmp/" ``` ### CLI Input Prefix Worker -Same as the [Multiple CLI Input Worker](#multiple-cli-input-worker). However the CLI application might need a pre and suffix. +Same as the [Multiple CLI Input Worker](#multiple-cli-input-worker). However, the CLI application might need a pre and +suffix. + +Something like -Something like ```bash $ cli-script.sh start Your Input: PREFIX QUERY SUFFIX @@ -301,33 +300,31 @@ QUERY RESULT 2 ``` +Parameters are : -Parameters are : - -| parameter | optional | default | description | -| ----- | --- | ----- | --------- | -| queriesFile | no | | File containg the queries this worker should use. | -| initFinished | no | | String which occurs when the application is ready to be requested (e.g. after loading) | -| queryFinished | no | | String which occurs if the query response finished | -| queryError | no | | String which occurs when an error during the query execution happend | -| queryPrefix | no | | String to use as a PREFIX before the query. | -| querySuffix | no | | String to use as a SUFFIX after the query. | -| numberOfProcesses | yes | 5 | The number of times the application should be started to assure a smooth benchmark. see [Multiple CLI Input Worker](#multiple-cli-input-worker). | -| timeOut | yes | 180000 (3 minutes) | The timeout in MS after a query should be aborted | -| fixedLatency | yes | 0 | If the value (in MS) should be waited between each query. Simulating network latency or user behaviour. | -| gaussianLatency | yes | 0 | A random value between `[0, 2*value]` (in MS) will be waited between each query. Simulating network latency or user behaviour. | - +| parameter | optional | default | description | +|-------------------|----------|--------------------|--------------------------------------------------------------------------------------------------------------------------------------------------| +| initFinished | no | | String which occurs when the application is ready to be requested (e.g. after loading) | +| queryFinished | no | | String which occurs if the query response finished | +| queryError | no | | String which occurs when an error during the query execution happend | +| queryPrefix | no | | String to use as a PREFIX before the query. | +| querySuffix | no | | String to use as a SUFFIX after the query. | +| numberOfProcesses | yes | 5 | The number of times the application should be started to assure a smooth benchmark. see [Multiple CLI Input Worker](#multiple-cli-input-worker). | An Example: + ```yaml - ... - workers: - - threads: 1 - className: "CLIInputPrefixWorker" - initFinished: "loading finished" - queryFinished: "query execution took:" - queryError: "Error happend during request" - queryPrefix: "SPARQL" - querySuffix: ";" + ... + workers: + - threads: 1 + className: "CLIInputPrefixWorker" + queries: + ... + initFinished: "loading finished" + queryFinished: "query execution took:" + queryError: "Error happened during request" + queryPrefix: "SPARQL" + querySuffix: ";" ``` + Will send the following as Input `SPARQL QUERY ;` diff --git a/example-suite.yml b/example-suite.yml index c4290113e..eef144436 100644 --- a/example-suite.yml +++ b/example-suite.yml @@ -27,23 +27,22 @@ tasks: warmup: # 1 minutes (is in ms) timeLimit: 600000 - # queryHandler could be set too, same as in the stresstest configuration, otherwise the same queryHandler will be use. - # workers are set the same way as in the configuration part workers: - threads: 1 - className: "SPARQLWorker" - queriesFile: "queries_warmup.txt" + className: "HttpGetWorker" + queries: + location: "queries_warmup.txt" timeOut: 180000 - queryHandler: - className: "InstancesQueryHandler" workers: - threads: 16 - className: "SPARQLWorker" - queriesFile: "queries_easy.txt" + className: "HttpGetWorker" + queries: + location: "queries_easy.txt" timeOut: 180000 - threads: 4 - className: "SPARQLWorker" - queriesFile: "queries_complex.txt" + className: "HttpGetWorker" + queries: + location: "queries_complex.txt" fixedLatency: 100 gaussianLatency: 50 parameterName: "query" diff --git a/pom.xml b/pom.xml index a235a22cd..c3eaf07f7 100644 --- a/pom.xml +++ b/pom.xml @@ -39,9 +39,9 @@ ${major.minor.version}.${build.version} ${major.version}.${minor.version} - 3 - 3 - 2 + 4 + 0 + 0 diff --git a/schema/iguana.owl b/schema/iguana.owl index ef83ff1a7..296947e0e 100644 --- a/schema/iguana.owl +++ b/schema/iguana.owl @@ -26,11 +26,11 @@ xmlns:iont="http://iguana-benchmark.eu/class/" > - + Iguana results ontology - 3.3.2 + 4.0.0 2020/09/18 - 2022/09/23 + 2022/11/07 Iguana results ontology The Iguana results ontology explains the rdf results of an Iguana benchmark. From 3be9745e87b9ff3aed15260172bfc79353e5e9b0 Mon Sep 17 00:00:00 2001 From: frensing Date: Sun, 13 Nov 2022 22:33:28 +0200 Subject: [PATCH 27/47] javadoc --- .../iguana/cc/query/handler/QueryHandler.java | 32 +++++++++++++++++++ .../cc/query/pattern/PatternHandler.java | 21 ++++++++++++ .../query/selector/AbstractQuerySelector.java | 5 +++ .../cc/query/selector/QuerySelector.java | 6 ++++ .../selector/impl/LinearQuerySelector.java | 10 ++++-- .../selector/impl/RandomQuerySelector.java | 9 ++++-- .../iguana/cc/query/set/AbstractQuerySet.java | 6 ++++ .../aksw/iguana/cc/query/set/QuerySet.java | 2 +- .../cc/query/set/impl/FileBasedQuerySet.java | 5 +++ .../cc/query/set/impl/InMemQuerySet.java | 6 ++++ .../cc/query/source/AbstractQuerySource.java | 6 ++++ .../iguana/cc/query/source/QuerySource.java | 6 ++++ .../source/impl/FileLineQuerySource.java | 5 +++ .../source/impl/FileSeparatorQuerySource.java | 6 ++++ .../query/source/impl/FolderQuerySource.java | 6 ++++ 15 files changed, 126 insertions(+), 5 deletions(-) diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/handler/QueryHandler.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/handler/QueryHandler.java index 6ffda81df..a10fab655 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/handler/QueryHandler.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/handler/QueryHandler.java @@ -24,6 +24,14 @@ import java.util.List; import java.util.Map; +/** + * This QueryHandler which is used by every worker who is extending the AbstractWorker. + * It initializes the QuerySource, QuerySelector, QuerySet and, if needed, PatternHandler. + * After the initialization, it provides the next query to the worker using the generated QuerySource + * and the order given by the QuerySelector. + * + * @author frensing + */ public class QueryHandler { protected final Logger LOGGER = LoggerFactory.getLogger(QueryHandler.class); @@ -90,6 +98,11 @@ public LanguageProcessor getLanguageProcessor() { return this.langProcessor; } + /** + * Will initialize the PatternHandler if a pattern config is given. + * The PatternHandler uses the original query source and generates a new query source + * which is used during the benchmark execution. + */ private void initPattern() { Map patternConfig = (Map) this.config.get("pattern"); PatternHandler patternHandler = new PatternHandler(patternConfig, createQuerySource()); @@ -97,6 +110,12 @@ private void initPattern() { initQuerySet(patternHandler.generateQuerySource()); } + /** + * Will initialize the QuerySet. + * If caching is not set or set to true, the InMemQuerySet will be used. Otherwise the FileBasedQuerySet. + * + * @param querySource The QuerySource which contains the queries. + */ private void initQuerySet(QuerySource querySource) { this.caching = (Boolean) this.config.getOrDefault("caching", true); @@ -111,6 +130,14 @@ private void initQuerySet() { initQuerySet(createQuerySource()); } + /** + * Will initialize the QuerySource. + * Depending on the format configuration, the FileLineQuerySource, + * FileSeparatorQuerySource or FolderQuerySource will be used. + * The FileSeparatorQuerySource can be further configured with a separator. + * + * @return The QuerySource which contains the queries. + */ private QuerySource createQuerySource() { Object formatObj = this.config.getOrDefault("format", "one-per-line"); if (formatObj instanceof Map) { @@ -132,6 +159,11 @@ private QuerySource createQuerySource() { return null; } + /** + * Will initialize the QuerySelector that provides the next query index during the benchmark execution. + *

+ * currently linear or random (with seed) are implemented + */ private void initQuerySelector() { Object orderObj = this.config.getOrDefault("order", "linear"); diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/pattern/PatternHandler.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/pattern/PatternHandler.java index 76e2b74ca..9573beed3 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/pattern/PatternHandler.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/pattern/PatternHandler.java @@ -12,6 +12,16 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; +/** + * This class is used to instantiate SPARQL pattern queries. + * It will replace create a SPARQL query to request values, for the variables + * in the pattern query, from a provided SPARQL endpoint. + *

+ * The instantiated queries are located in a text file, which is created at the given location. + * If a fitting query file is already present, the queries will not be instantiated again. + * + * @author frensing + */ public class PatternHandler { private static final Logger LOGGER = LoggerFactory.getLogger(PatternHandler.class); @@ -28,6 +38,13 @@ public PatternHandler(Map config, QuerySource querySource) { init(); } + /** + * This method will generate the queries from the given patterns, writes them + * to a file, and returns a QuerySource based on that file. + * The QuerySource is then used in the QueryHandler to get the queries. + * + * @return QuerySource containing the instantiated queries + */ public QuerySource generateQuerySource() { File cacheFile = new File(this.outputFolder + File.separator + this.querySource.hashCode()); if (cacheFile.exists()) { @@ -55,6 +72,10 @@ public QuerySource generateQuerySource() { return new FileLineQuerySource(cacheFile.getAbsolutePath()); } + /** + * Initializes the PatternHandler + * Sets up the output folder, the endpoint and the limit. + */ private void init() { this.endpoint = (String) this.config.get("endpoint"); if (this.endpoint == null) { diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/selector/AbstractQuerySelector.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/selector/AbstractQuerySelector.java index b3d77d51b..044412ff2 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/selector/AbstractQuerySelector.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/selector/AbstractQuerySelector.java @@ -1,5 +1,10 @@ package org.aksw.iguana.cc.query.selector; +/** + * The QuerySelector is used to get the next query index. + * + * @author frensing + */ public abstract class AbstractQuerySelector implements QuerySelector { protected int size; diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/selector/QuerySelector.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/selector/QuerySelector.java index 27f139e4b..82fc7c66b 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/selector/QuerySelector.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/selector/QuerySelector.java @@ -1,5 +1,11 @@ package org.aksw.iguana.cc.query.selector; +/** + * The QuerySelector that is used to get the next query index. + * It is used by the QueryHandler to get the next query. + * + * @author frensing + */ public interface QuerySelector { /** * This method gives the next query index that should be used. diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/selector/impl/LinearQuerySelector.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/selector/impl/LinearQuerySelector.java index 989eddd26..ee763b8bd 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/selector/impl/LinearQuerySelector.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/selector/impl/LinearQuerySelector.java @@ -1,9 +1,15 @@ package org.aksw.iguana.cc.query.selector.impl; import org.aksw.iguana.cc.query.selector.AbstractQuerySelector; -import org.aksw.iguana.commons.annotation.Shorthand; -@Shorthand("LinearQuerySelector") +/** + * This QuerySelector is used to get the next query index in a linear order. + * If the last query is reached it starts again at the first query. + *

+ * It is used by the QueryHandler to get the next query. + * + * @author frensing + */ public class LinearQuerySelector extends AbstractQuerySelector { protected int querySelector; diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/selector/impl/RandomQuerySelector.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/selector/impl/RandomQuerySelector.java index c835f894b..6cd1e4a3e 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/selector/impl/RandomQuerySelector.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/selector/impl/RandomQuerySelector.java @@ -1,11 +1,16 @@ package org.aksw.iguana.cc.query.selector.impl; import org.aksw.iguana.cc.query.selector.AbstractQuerySelector; -import org.aksw.iguana.commons.annotation.Shorthand; import java.util.Random; -@Shorthand("RandomQuerySelector") +/** + * This QuerySelector is used to get the next query index in a random order. + *

+ * It is used by the QueryHandler to get the next query. + * + * @author frensing + */ public class RandomQuerySelector extends AbstractQuerySelector { protected Random querySelector; diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/AbstractQuerySet.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/AbstractQuerySet.java index 3c81a80e9..df55eada2 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/AbstractQuerySet.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/AbstractQuerySet.java @@ -2,6 +2,12 @@ import org.aksw.iguana.cc.query.source.QuerySource; +/** + * The abstract class for a QuerySet. + * It implements the basic functions that are shared for all QuerySets. + * + * @author frensing + */ public abstract class AbstractQuerySet implements QuerySet { protected QuerySource querySource; diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/QuerySet.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/QuerySet.java index c873635ce..213c85d59 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/QuerySet.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/QuerySet.java @@ -4,7 +4,7 @@ import java.io.IOException; /** - * A query set contains a benchmark query (this might be several queries in itself) + * A query set provides the Queries to the QueryHandler. */ public interface QuerySet { diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/impl/FileBasedQuerySet.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/impl/FileBasedQuerySet.java index 09a1cb24b..ecba5aa13 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/impl/FileBasedQuerySet.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/impl/FileBasedQuerySet.java @@ -5,6 +5,11 @@ import java.io.IOException; +/** + * A query set which reads the queries directly from a file. + * + * @author frensing + */ public class FileBasedQuerySet extends AbstractQuerySet { public FileBasedQuerySet(String name, QuerySource querySource) { diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/impl/InMemQuerySet.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/impl/InMemQuerySet.java index 216a84375..db22d57cc 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/impl/InMemQuerySet.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/impl/InMemQuerySet.java @@ -8,6 +8,12 @@ import java.io.IOException; import java.util.List; +/** + * A query set which reads the queries into memory on initialization. + * During the benchmark the query are returned from the memory. + * + * @author frensing + */ public class InMemQuerySet extends AbstractQuerySet { private static final Logger LOGGER = LoggerFactory.getLogger(InMemQuerySet.class); diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/AbstractQuerySource.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/AbstractQuerySource.java index 9b70065c5..2c15b0f6a 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/AbstractQuerySource.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/AbstractQuerySource.java @@ -2,6 +2,12 @@ import org.aksw.iguana.cc.utils.FileUtils; +/** + * The abstract class for a QuerySource. + * It implements the basic functions that are shared for all QuerySources. + * + * @author frensing + */ public abstract class AbstractQuerySource implements QuerySource { protected String path; diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/QuerySource.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/QuerySource.java index 8031b5fa6..2c0a5b210 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/QuerySource.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/QuerySource.java @@ -3,6 +3,12 @@ import java.io.IOException; import java.util.List; +/** + * The QuerySource is providing the queries to the QuerySet. + * It abstracts the actual format of the query files. + * + * @author frensing + */ public interface QuerySource { /** diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/impl/FileLineQuerySource.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/impl/FileLineQuerySource.java index 37e92e062..67b839597 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/impl/FileLineQuerySource.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/impl/FileLineQuerySource.java @@ -10,6 +10,11 @@ import java.nio.file.Files; import java.util.List; +/** + * The FileLineQuerySource reads queries from a file with one query per line. + * + * @author frensing + */ public class FileLineQuerySource extends AbstractQuerySource { private static final Logger LOGGER = LoggerFactory.getLogger(FileLineQuerySource.class); diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/impl/FileSeparatorQuerySource.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/impl/FileSeparatorQuerySource.java index 7f2756019..f96604921 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/impl/FileSeparatorQuerySource.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/impl/FileSeparatorQuerySource.java @@ -15,6 +15,12 @@ import java.util.List; import java.util.stream.Stream; +/** + * The FileSeparatorQuerySource reads queries from a file with + * (multiline) queries that are separated by a separator line. + * + * @author frensing + */ public class FileSeparatorQuerySource extends AbstractQuerySource { private static final Logger LOGGER = LoggerFactory.getLogger(FileSeparatorQuerySource.class); diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/impl/FolderQuerySource.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/impl/FolderQuerySource.java index 8581c4f94..0075b0047 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/impl/FolderQuerySource.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/impl/FolderQuerySource.java @@ -12,6 +12,12 @@ import java.util.ArrayList; import java.util.List; +/** + * The FileSeparatorQuerySource reads queries from a folder with query files. + * Each query contains one (multiline) query. + * + * @author frensing + */ public class FolderQuerySource extends AbstractQuerySource { protected static final Logger LOGGER = LoggerFactory.getLogger(FolderQuerySource.class); From e9b40b183a3c8323eb4966eeed7b0aa6d991d81a Mon Sep 17 00:00:00 2001 From: frensing Date: Sun, 13 Nov 2022 23:29:02 +0200 Subject: [PATCH 28/47] refactor TypedFactory create method --- .../iguana/commons/factory/TypedFactory.java | 109 +++++++++--------- .../commons/factory/TypedFactoryTest.java | 77 +++++++------ .../aksw/iguana/cc/config/IguanaConfig.java | 2 +- .../cc/config/elements/MetricConfig.java | 10 +- .../cc/config/elements/StorageConfig.java | 8 +- .../aksw/iguana/cc/config/elements/Task.java | 4 +- .../iguana/cc/query/handler/QueryHandler.java | 4 +- .../aksw/iguana/cc/tasks/impl/Stresstest.java | 20 ++-- .../aksw/iguana/cc/worker/AbstractWorker.java | 2 +- .../cc/worker/impl/CLIInputFileWorker.java | 2 +- .../cc/worker/impl/CLIInputPrefixWorker.java | 2 +- .../iguana/cc/worker/impl/CLIInputWorker.java | 2 +- .../aksw/iguana/cc/worker/impl/CLIWorker.java | 2 +- .../iguana/cc/worker/impl/HttpGetWorker.java | 2 +- .../iguana/cc/worker/impl/HttpPostWorker.java | 2 +- .../iguana/cc/worker/impl/HttpWorker.java | 2 +- .../worker/impl/MultipleCLIInputWorker.java | 2 +- .../iguana/cc/worker/impl/SPARQLWorker.java | 6 +- .../iguana/cc/worker/impl/UPDATEWorker.java | 2 +- .../cc/query/handler/QueryHandlerTest.java | 2 +- .../iguana/cc/tasks/impl/StresstestTest.java | 10 +- .../aksw/iguana/cc/worker/HTTPWorkerTest.java | 2 +- .../aksw/iguana/cc/worker/MockupWorker.java | 4 +- .../iguana/cc/worker/UPDATEWorkerTest.java | 18 +-- .../cc/worker/impl/CLIWorkersTests.java | 4 +- .../cc/worker/impl/HttpPostWorkerTest.java | 4 +- 26 files changed, 151 insertions(+), 153 deletions(-) diff --git a/iguana.commons/src/main/java/org/aksw/iguana/commons/factory/TypedFactory.java b/iguana.commons/src/main/java/org/aksw/iguana/commons/factory/TypedFactory.java index 8d62570f0..4385f8a52 100644 --- a/iguana.commons/src/main/java/org/aksw/iguana/commons/factory/TypedFactory.java +++ b/iguana.commons/src/main/java/org/aksw/iguana/commons/factory/TypedFactory.java @@ -42,7 +42,6 @@ private String getClassName(String className){ * class name and the constructor arguments, be aware that all arguments * must be Strings in the constructor. * - * * @param className * The Class Name of the Implemented T Object * @param constructorArgs @@ -57,9 +56,7 @@ public T create(String className, Object[] constructorArgs){ constructorArgs2 = new Object[0]; } Class[] stringClass = new Class[constructorArgs2.length]; - for(int i=0;i[] constructorClasses) { + public T create(String className, Object[] constructorArgs, Class[] constructorClasses) { Object[] constructorArgs2 = constructorArgs; - + if (className == null) { return null; } @@ -102,9 +97,7 @@ public T create(String className, } if(constructorClasses==null){ constructorClasses = new Class[constructorArgs2.length]; - for (int i = 0; i < constructorClasses.length; i++) { - constructorClasses[i] = String.class; - } + Arrays.fill(constructorClasses, String.class); } try { @@ -133,19 +126,19 @@ public T create(String className, * @param map key-value pair, whereas key represents the parameter name, where as value will be the value of the instantiation * @return The instantiated object or null no constructor was found */ - public T create(String className, Map map){ + public T create(String className, Map map) { Class clazz; - if(className==null){ + if (className == null) { return null; } try { - clazz = (Class) ClassLoader - .getSystemClassLoader().loadClass(getClassName(className)); + clazz = (Class) ClassLoader.getSystemClassLoader().loadClass(getClassName(className)); } catch (ClassNotFoundException e1) { return null; } - Constructor[] constructors = clazz.getConstructors(); - find: for(Constructor constructor : constructors){ + Constructor[] constructors = clazz.getConstructors(); + find: + for (Constructor constructor : constructors) { //ParameterNames would be a backup //ParameterNames paramNames = (ParameterNames) constructor.getAnnotation(ParameterNames.class); //if(paramNames==null){ @@ -153,25 +146,27 @@ public T create(String className, Map map){ //} Parameter[] params = constructor.getParameters(); - List names = new ArrayList(); - List types = new ArrayList(); - Set canBeNull = new HashSet(); - for(Parameter p : params){ + List names = new ArrayList<>(); + List> types = new ArrayList<>(); + Set canBeNull = new HashSet<>(); + for (Parameter p : params) { names.add(p.getName()); types.add(p.getType()); - if(p.isAnnotationPresent(Nullable.class)){ + if (p.isAnnotationPresent(Nullable.class)) { canBeNull.add(p.getName()); } } - List instanceNames = new ArrayList(map.keySet()); + List instanceNames = new ArrayList<>(map.keySet()); Object[] constructorArgs = new Object[names.size()]; - if(!checkIfFits(map, names, canBeNull)){continue;} - for(Object key : instanceNames){ + if (!checkIfFits(map, names, canBeNull)) { + continue; + } + for (String key : instanceNames) { Object value = map.get(key); //Check if constructor can map keys to param Names - int indexKey = names.indexOf(key.toString()); + int indexKey = names.indexOf(key); Class clazz2 = types.get(indexKey); - if(!clazz2.isInstance(value)){ + if (!clazz2.isInstance(value)) { continue find; } constructorArgs[indexKey] = value; @@ -195,26 +190,26 @@ public T create(String className, Map map){ /** * Checks if the giving parameter key-value mapping fits the constructor parameter names (key vs names) and takes into account that the parameter is allowed to be null and thus * can be disregarded - * @param map paramater - Object Map - * @param names parameter names of the actual constructor + * + * @param map paramater - Object Map + * @param names parameter names of the actual constructor * @param canBeNull all paramaters who can be null * @return true if constructor fits, otherwise false */ - private boolean checkIfFits(Map map, List names, Set canBeNull) { + private boolean checkIfFits(Map map, List names, Set canBeNull) { //check if all provided parameter names are in the constructor - for(Object key : map.keySet()){ - Object value = map.get(key); - if(!names.contains(key.toString())){ + for (String key : map.keySet()) { + if (!names.contains(key)) { return false; } } //check if all notNull objects are provided - Set keySet = map.keySet(); - for(String name : names){ + Set keySet = map.keySet(); + for (String name : names) { //we can safely assume that Object is string - if(!keySet.contains(name)){ + if (!keySet.contains(name)) { //check if parameter is Nullable - if(!canBeNull.contains(name)){ + if (!canBeNull.contains(name)) { return false; } } @@ -235,42 +230,44 @@ private boolean checkIfFits(Map map, List names, Set map){ + public T createAnnotated(String className, Map map) { Class clazz; try { - clazz = (Class) ClassLoader - .getSystemClassLoader().loadClass(getClassName(className)); + clazz = (Class) ClassLoader.getSystemClassLoader().loadClass(getClassName(className)); } catch (ClassNotFoundException e1) { return null; } - Constructor[] constructors = clazz.getConstructors(); - find: for(Constructor constructor : constructors){ - ParameterNames paramNames = (ParameterNames) constructor.getAnnotation(ParameterNames.class); - if(paramNames==null){ - continue ; + Constructor[] constructors = clazz.getConstructors(); + find: + for (Constructor constructor : constructors) { + ParameterNames paramNames = constructor.getAnnotation(ParameterNames.class); + if (paramNames == null) { + continue; } Parameter[] params = constructor.getParameters(); - List names = new ArrayList(); - List types = new ArrayList(); - Set canBeNull = new HashSet(); - for(int i=0;i names = new ArrayList<>(); + List> types = new ArrayList<>(); + Set canBeNull = new HashSet<>(); + for (int i = 0; i < params.length; i++) { Parameter p = params[i]; names.add(paramNames.names()[i]); types.add(p.getType()); - if(p.isAnnotationPresent(Nullable.class)){ + if (p.isAnnotationPresent(Nullable.class)) { canBeNull.add(p.getName()); } } - List instanceNames = new ArrayList(map.keySet()); + List instanceNames = new ArrayList<>(map.keySet()); Object[] constructorArgs = new Object[names.size()]; - if(!checkIfFits(map, names, canBeNull)){continue;} - for(Object key : instanceNames){ + if (!checkIfFits(map, names, canBeNull)) { + continue; + } + for (String key : instanceNames) { Object value = map.get(key); //Check if constructor can map keys to param Names - int indexKey = names.indexOf(key.toString()); + int indexKey = names.indexOf(key); Class clazz2 = types.get(indexKey); - if(!clazz2.isInstance(value)){ + if (!clazz2.isInstance(value)) { continue find; } constructorArgs[indexKey] = value; diff --git a/iguana.commons/src/test/java/org/aksw/iguana/commons/factory/TypedFactoryTest.java b/iguana.commons/src/test/java/org/aksw/iguana/commons/factory/TypedFactoryTest.java index ff8ac9a92..3e27832dd 100644 --- a/iguana.commons/src/test/java/org/aksw/iguana/commons/factory/TypedFactoryTest.java +++ b/iguana.commons/src/test/java/org/aksw/iguana/commons/factory/TypedFactoryTest.java @@ -6,16 +6,17 @@ import java.util.Map; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; public class TypedFactoryTest { @Test public void argumentClassesTest() { - String[] args = new String[] { "a1", "b1" }; - String[] args2 = new String[] { "a2", "b2" }; - TypedFactory factory = new TypedFactory(); + String[] args = new String[]{"a1", "b1"}; + String[] args2 = new String[]{"a2", "b2"}; + TypedFactory factory = new TypedFactory<>(); FactorizedObject testObject = factory.create("org.aksw.iguana.commons.factory.FactorizedObject", - new Object[] { args, args2 }, new Class[] { String[].class, String[].class }); + new Object[]{args, args2}, new Class[]{String[].class, String[].class}); assertEquals(args[0], testObject.getArgs()[0]); assertEquals(args[1], testObject.getArgs()[1]); assertEquals(args2[0], testObject.getArgs2()[0]); @@ -25,58 +26,58 @@ public void argumentClassesTest() { @Test - public void noConstructor(){ - TypedFactory factory = new TypedFactory(); - HashMap map = new HashMap(); + public void noConstructor() { + TypedFactory factory = new TypedFactory<>(); + HashMap map = new HashMap<>(); map.put("nope", "nope"); - assertEquals(null, factory.create("org.aksw.iguana.commons.factory.FactorizedObject", map)); - assertEquals(null, factory.create("org.aksw.iguana.commons.factory.FactorizedObject", new Object[]{"nope"})); - assertEquals(null, factory.create("org.aksw.iguana.commons.factory.FactorizedObject", new Object[]{"nope"}, new Class[]{String.class})); - assertEquals(null, factory.createAnnotated("org.aksw.iguana.commons.factory.AnnotatedFactorizedObject", map)); + assertNull(factory.create("org.aksw.iguana.commons.factory.FactorizedObject", map)); + assertNull(factory.create("org.aksw.iguana.commons.factory.FactorizedObject", new Object[]{"nope"})); + assertNull(factory.create("org.aksw.iguana.commons.factory.FactorizedObject", new Object[]{"nope"}, new Class[]{String.class})); + assertNull(factory.createAnnotated("org.aksw.iguana.commons.factory.AnnotatedFactorizedObject", map)); map.clear(); map.put("a", 123); map.put("b", true); - assertEquals(null, factory.create("org.aksw.iguana.commons.factory.FactorizedObject", map)); - assertEquals(null, factory.createAnnotated("org.aksw.iguana.commons.factory.AnnotatedFactorizedObject", map)); + assertNull(factory.create("org.aksw.iguana.commons.factory.FactorizedObject", map)); + assertNull(factory.createAnnotated("org.aksw.iguana.commons.factory.AnnotatedFactorizedObject", map)); } @Test - public void nullConstructorClass(){ - TypedFactory factory = new TypedFactory(); - FactorizedObject testObject = factory.create("org.aksw.iguana.commons.factory.FactorizedObject", new Object[]{"a", "b", "c"}, (Class[])null); + public void nullConstructorClass() { + TypedFactory factory = new TypedFactory<>(); + FactorizedObject testObject = factory.create("org.aksw.iguana.commons.factory.FactorizedObject", new Object[]{"a", "b", "c"}, null); assertEquals("a", testObject.getArgs()[0]); assertEquals("b", testObject.getArgs()[1]); assertEquals("c", testObject.getArgs()[2]); - testObject = factory.create("org.aksw.iguana.commons.factory.FactorizedObject", (Object[])null, (Class[])null); + testObject = factory.create("org.aksw.iguana.commons.factory.FactorizedObject", null, null); assertEquals("a3", testObject.getArgs()[0]); assertEquals("b3", testObject.getArgs()[1]); } @Test - public void nullClass(){ - TypedFactory factory = new TypedFactory(); - assertEquals(null, factory.create(null, new HashMap<>())); - assertEquals(null, factory.create(null, new Object[]{})); - assertEquals(null, factory.create(null, new Object[]{}, new Class[]{})); + public void nullClass() { + TypedFactory factory = new TypedFactory<>(); + assertNull(factory.create(null, new HashMap<>())); + assertNull(factory.create(null, new Object[]{})); + assertNull(factory.create(null, new Object[]{}, new Class[]{})); } @Test - public void classNameNotFoundTest(){ - TypedFactory factory = new TypedFactory(); - assertEquals(null, factory.create("thisClassShouldNotExist", new HashMap<>())); - assertEquals(null, factory.create("thisClassShouldNotExist", new Object[]{})); - assertEquals(null, factory.create("thisClassShouldNotExist", new Object[]{}, new Class[]{})); - assertEquals(null, factory.createAnnotated("thisClassShouldNotExist", new HashMap<>())); + public void classNameNotFoundTest() { + TypedFactory factory = new TypedFactory<>(); + assertNull(factory.create("thisClassShouldNotExist", new HashMap<>())); + assertNull(factory.create("thisClassShouldNotExist", new Object[]{})); + assertNull(factory.create("thisClassShouldNotExist", new Object[]{}, new Class[]{})); + assertNull(factory.createAnnotated("thisClassShouldNotExist", new HashMap<>())); } @Test public void argumentStringsTest() { - TypedFactory factory = new TypedFactory(); - FactorizedObject testObject = factory.create("org.aksw.iguana.commons.factory.FactorizedObject", (Object[])null); + TypedFactory factory = new TypedFactory<>(); + FactorizedObject testObject = factory.create("org.aksw.iguana.commons.factory.FactorizedObject", (Object[]) null); assertEquals("a3", testObject.getArgs()[0]); assertEquals("b3", testObject.getArgs()[1]); } @@ -85,8 +86,8 @@ public void argumentStringsTest() { @Test public void mapCreationTestParameterNames() { - TypedFactory factory = new TypedFactory(); - Map arguments = new HashMap(); + TypedFactory factory = new TypedFactory<>(); + Map arguments = new HashMap<>(); arguments.put("a", "a4"); arguments.put("b", "b4"); arguments.put("c", "c4"); @@ -104,8 +105,8 @@ public void mapCreationTestParameterNames() { @Test public void testNullable() { - TypedFactory factory = new TypedFactory(); - Map arguments = new HashMap(); + TypedFactory factory = new TypedFactory<>(); + Map arguments = new HashMap<>(); arguments.put("a", "a4"); arguments.put("b", "b4"); FactorizedObject testObject = factory.create("org.aksw.iguana.commons.factory.FactorizedObject", arguments); @@ -121,8 +122,8 @@ public void testNullable() { @Test public void mapCreationTest() { - TypedFactory factory = new TypedFactory(); - Map arguments = new HashMap(); + TypedFactory factory = new TypedFactory<>(); + Map arguments = new HashMap<>(); arguments.put("a", "a4"); arguments.put("b", "b4"); arguments.put("c", "c4"); @@ -136,8 +137,8 @@ public void mapCreationTest() { @Test public void shortHandAnnotationTest() { - TypedFactory factory = new TypedFactory(); - Map arguments = new HashMap(); + TypedFactory factory = new TypedFactory<>(); + Map arguments = new HashMap<>(); arguments.put("a", "a4"); arguments.put("b", "b4"); arguments.put("c", "c4"); diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/config/IguanaConfig.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/config/IguanaConfig.java index d6a4dfa68..222f22dca 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/config/IguanaConfig.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/config/IguanaConfig.java @@ -131,7 +131,7 @@ private RPController initResultProcessor() { metrics.add(config); config = new MetricConfig(); config.setClassName(QPSMetric.class.getCanonicalName()); - Map configMap = new HashMap<>(); + Map configMap = new HashMap<>(); configMap.put("penalty", 180000); config.setConfiguration(configMap); metrics.add(config); diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/config/elements/MetricConfig.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/config/elements/MetricConfig.java index 3e2d5a37d..ac504ca1c 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/config/elements/MetricConfig.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/config/elements/MetricConfig.java @@ -15,8 +15,8 @@ public class MetricConfig { @JsonProperty(required = true) private String className; - @JsonProperty(required = false) - private Map configuration = new HashMap(); + @JsonProperty() + private Map configuration = new HashMap<>(); public String getClassName() { @@ -27,16 +27,16 @@ public void setClassName(String className) { this.className = className; } - public Map getConfiguration() { + public Map getConfiguration() { return configuration; } - public void setConfiguration(Map configuration) { + public void setConfiguration(Map configuration) { this.configuration = configuration; } public Metric createMetric() { - TypedFactory factory = new TypedFactory(); + TypedFactory factory = new TypedFactory<>(); return factory.create(className, configuration); } } diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/config/elements/StorageConfig.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/config/elements/StorageConfig.java index 9144800ec..60226c300 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/config/elements/StorageConfig.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/config/elements/StorageConfig.java @@ -17,7 +17,7 @@ public class StorageConfig { private String className; @JsonProperty - private Map configuration = new HashMap(); + private Map configuration = new HashMap<>(); public String getClassName() { return className; @@ -27,16 +27,16 @@ public void setClassName(String className) { this.className = className; } - public Map getConfiguration() { + public Map getConfiguration() { return configuration; } - public void setConfiguration(Map configuration) { + public void setConfiguration(Map configuration) { this.configuration = configuration; } public Storage createStorage() { - TypedFactory factory = new TypedFactory(); + TypedFactory factory = new TypedFactory<>(); return factory.create(className, configuration); } } diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/config/elements/Task.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/config/elements/Task.java index 036c6457b..1dd108934 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/config/elements/Task.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/config/elements/Task.java @@ -12,7 +12,7 @@ public class Task implements Serializable { @JsonProperty(required = true) - private Map configuration = new HashMap<>(); + private Map configuration = new HashMap<>(); @JsonProperty(required = true) private String className; @@ -28,7 +28,7 @@ public void setName(String name) { this.name = name; } - public Map getConfiguration() { + public Map getConfiguration() { return configuration; } diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/handler/QueryHandler.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/handler/QueryHandler.java index a10fab655..d7b653988 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/handler/QueryHandler.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/handler/QueryHandler.java @@ -36,7 +36,7 @@ public class QueryHandler { protected final Logger LOGGER = LoggerFactory.getLogger(QueryHandler.class); - protected Map config; + protected Map config; protected Integer workerID; protected String location; protected int hashcode; @@ -49,7 +49,7 @@ public class QueryHandler { protected LanguageProcessor langProcessor; - public QueryHandler(Map config, Integer workerID) { + public QueryHandler(Map config, Integer workerID) { this.config = config; this.workerID = workerID; diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/tasks/impl/Stresstest.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/tasks/impl/Stresstest.java index b8ededdc9..e89faee65 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/tasks/impl/Stresstest.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/tasks/impl/Stresstest.java @@ -33,9 +33,9 @@ public class Stresstest extends AbstractTask { private static final Logger LOGGER = LoggerFactory.getLogger(Stresstest.class); - private final Map warmupConfig; + private final Map warmupConfig; private final List warmupWorkers = new ArrayList<>(); - private final List> workerConfig; + private final List> workerConfig; protected List workers = new LinkedList<>(); private Double warmupTimeMS; private Double timeLimit; @@ -43,21 +43,21 @@ public class Stresstest extends AbstractTask { private Instant startTime; - public Stresstest(Integer timeLimit, List> workers) { + public Stresstest(Integer timeLimit, List> workers) { this(timeLimit, workers, null); } - public Stresstest(Integer timeLimit, List> workers, Map warmup) { + public Stresstest(Integer timeLimit, List> workers, Map warmup) { this.timeLimit = timeLimit.doubleValue(); this.workerConfig = workers; this.warmupConfig = warmup; } - public Stresstest(List> workers, Integer noOfQueryMixes) { + public Stresstest(List> workers, Integer noOfQueryMixes) { this(workers, null, noOfQueryMixes); } - public Stresstest(List> workers, Map warmup, Integer noOfQueryMixes) { + public Stresstest(List> workers, Map warmup, Integer noOfQueryMixes) { this.noOfQueryMixes = noOfQueryMixes.longValue(); this.workerConfig = workers; this.warmupConfig = warmup; @@ -73,7 +73,7 @@ private void initWorkers() { private void createWarmupWorkers() { this.warmupTimeMS = ((Integer) this.warmupConfig.get("timeLimit")).doubleValue(); - List> warmupWorkerConfig = (List>) this.warmupConfig.get("workers"); + List> warmupWorkerConfig = (List>) this.warmupConfig.get("workers"); createWorkers(warmupWorkerConfig, this.warmupWorkers, this.warmupTimeMS); } @@ -81,14 +81,14 @@ private void createWorkers() { createWorkers(this.workerConfig, this.workers, this.timeLimit); } - private void createWorkers(List> workers, List workersToAddTo, Double timeLimit) { + private void createWorkers(List> workers, List workersToAddTo, Double timeLimit) { int workerID = 0; - for (Map workerConfig : workers) { + for (Map workerConfig : workers) { workerID += createWorker(workerConfig, workersToAddTo, timeLimit, workerID); } } - private int createWorker(Map workerConfig, List workersToAddTo, Double timeLimit, Integer baseID) { + private int createWorker(Map workerConfig, List workersToAddTo, Double timeLimit, Integer baseID) { //let TypedFactory create from className and configuration String className = workerConfig.remove("className").toString(); //if shorthand classname is used, exchange to full classname diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/AbstractWorker.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/AbstractWorker.java index 9fa96cee4..4b760a69a 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/AbstractWorker.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/AbstractWorker.java @@ -71,7 +71,7 @@ public abstract class AbstractWorker implements Worker { private Random latencyRandomizer; private Long endAtNOQM = null; - public AbstractWorker(String taskID, Integer workerID, Connection connection, Map queries, @Nullable Integer timeLimit, @Nullable Integer timeOut, @Nullable Integer fixedLatency, @Nullable Integer gaussianLatency) { + public AbstractWorker(String taskID, Integer workerID, Connection connection, Map queries, @Nullable Integer timeLimit, @Nullable Integer timeOut, @Nullable Integer fixedLatency, @Nullable Integer gaussianLatency) { this.taskID = taskID; this.workerID = workerID; this.con = connection; diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/impl/CLIInputFileWorker.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/impl/CLIInputFileWorker.java index ca4335948..50515b43c 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/impl/CLIInputFileWorker.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/impl/CLIInputFileWorker.java @@ -22,7 +22,7 @@ public class CLIInputFileWorker extends MultipleCLIInputWorker { private final String dir; - public CLIInputFileWorker(String taskID, Integer workerID, Connection connection, Map queries, @Nullable Integer timeLimit, @Nullable Integer timeOut, @Nullable Integer fixedLatency, @Nullable Integer gaussianLatency, String initFinished, String queryFinished, String queryError, @Nullable Integer numberOfProcesses, String directory) { + public CLIInputFileWorker(String taskID, Integer workerID, Connection connection, Map queries, @Nullable Integer timeLimit, @Nullable Integer timeOut, @Nullable Integer fixedLatency, @Nullable Integer gaussianLatency, String initFinished, String queryFinished, String queryError, @Nullable Integer numberOfProcesses, String directory) { super(taskID, workerID, connection, queries, timeLimit, timeOut, fixedLatency, gaussianLatency, initFinished, queryFinished, queryError, numberOfProcesses); this.dir = directory; } diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/impl/CLIInputPrefixWorker.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/impl/CLIInputPrefixWorker.java index 281639236..19e37cdd3 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/impl/CLIInputPrefixWorker.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/impl/CLIInputPrefixWorker.java @@ -23,7 +23,7 @@ public class CLIInputPrefixWorker extends MultipleCLIInputWorker { private final String prefix; private final String suffix; - public CLIInputPrefixWorker(String taskID, Integer workerID, Connection connection, Map queries, @Nullable Integer timeLimit, @Nullable Integer timeOut, @Nullable Integer fixedLatency, @Nullable Integer gaussianLatency, String initFinished, String queryFinished, String queryError, @Nullable Integer numberOfProcesses, String queryPrefix, String querySuffix) { + public CLIInputPrefixWorker(String taskID, Integer workerID, Connection connection, Map queries, @Nullable Integer timeLimit, @Nullable Integer timeOut, @Nullable Integer fixedLatency, @Nullable Integer gaussianLatency, String initFinished, String queryFinished, String queryError, @Nullable Integer numberOfProcesses, String queryPrefix, String querySuffix) { super(taskID, workerID, connection, queries, timeLimit, timeOut, fixedLatency, gaussianLatency, initFinished, queryFinished, queryError, numberOfProcesses); this.prefix = queryPrefix; this.suffix = querySuffix; diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/impl/CLIInputWorker.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/impl/CLIInputWorker.java index aa067cc35..32f0b099f 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/impl/CLIInputWorker.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/impl/CLIInputWorker.java @@ -35,7 +35,7 @@ public class CLIInputWorker extends AbstractWorker { private final String error; private Process process; - public CLIInputWorker(String taskID, Integer workerID, Connection connection, Map queries, @Nullable Integer timeLimit, @Nullable Integer timeOut, @Nullable Integer fixedLatency, @Nullable Integer gaussianLatency, String initFinished, String queryFinished, String queryError) { + public CLIInputWorker(String taskID, Integer workerID, Connection connection, Map queries, @Nullable Integer timeLimit, @Nullable Integer timeOut, @Nullable Integer fixedLatency, @Nullable Integer gaussianLatency, String initFinished, String queryFinished, String queryError) { super(taskID, workerID, connection, queries, timeLimit, timeOut, fixedLatency, gaussianLatency); this.initFinished = initFinished; this.queryFinished = queryFinished; diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/impl/CLIWorker.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/impl/CLIWorker.java index b7ebc7506..e6315ec36 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/impl/CLIWorker.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/impl/CLIWorker.java @@ -32,7 +32,7 @@ public class CLIWorker extends AbstractWorker { private final Logger LOGGER = LoggerFactory.getLogger(getClass()); - public CLIWorker(String taskID, Integer workerID, Connection connection, Map queries, @Nullable Integer timeLimit, @Nullable Integer timeOut, @Nullable Integer fixedLatency, @Nullable Integer gaussianLatency) { + public CLIWorker(String taskID, Integer workerID, Connection connection, Map queries, @Nullable Integer timeLimit, @Nullable Integer timeOut, @Nullable Integer fixedLatency, @Nullable Integer gaussianLatency) { super(taskID, workerID, connection, queries, timeLimit, timeOut, fixedLatency, gaussianLatency); } diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/impl/HttpGetWorker.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/impl/HttpGetWorker.java index 0f81ed6f2..aa279de10 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/impl/HttpGetWorker.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/impl/HttpGetWorker.java @@ -24,7 +24,7 @@ public class HttpGetWorker extends HttpWorker { protected String responseType = null; - public HttpGetWorker(String taskID, Integer workerID, Connection connection, Map queries, @Nullable Integer timeLimit, @Nullable Integer timeOut, @Nullable Integer fixedLatency, @Nullable Integer gaussianLatency, @Nullable String parameterName, @Nullable String responseType) { + public HttpGetWorker(String taskID, Integer workerID, Connection connection, Map queries, @Nullable Integer timeLimit, @Nullable Integer timeOut, @Nullable Integer fixedLatency, @Nullable Integer gaussianLatency, @Nullable String parameterName, @Nullable String responseType) { super(taskID, workerID, connection, queries, timeLimit, timeOut, fixedLatency, gaussianLatency); if (parameterName != null) { diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/impl/HttpPostWorker.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/impl/HttpPostWorker.java index f46a44afe..2043b5bf6 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/impl/HttpPostWorker.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/impl/HttpPostWorker.java @@ -24,7 +24,7 @@ public class HttpPostWorker extends HttpGetWorker { private String contentType = "text/plain"; - public HttpPostWorker(String taskID, Integer workerID, Connection connection, Map queries, @Nullable Integer timeLimit, @Nullable Integer timeOut, @Nullable Integer fixedLatency, @Nullable Integer gaussianLatency, @Nullable String parameterName, @Nullable String responseType, @Nullable String contentType) { + public HttpPostWorker(String taskID, Integer workerID, Connection connection, Map queries, @Nullable Integer timeLimit, @Nullable Integer timeOut, @Nullable Integer fixedLatency, @Nullable Integer gaussianLatency, @Nullable String parameterName, @Nullable String responseType, @Nullable String contentType) { super(taskID, workerID, connection, queries, timeLimit, timeOut, fixedLatency, gaussianLatency, parameterName, responseType); if (parameterName == null) { diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/impl/HttpWorker.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/impl/HttpWorker.java index da6236d00..064280204 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/impl/HttpWorker.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/impl/HttpWorker.java @@ -50,7 +50,7 @@ public abstract class HttpWorker extends AbstractWorker { protected Instant requestStartTime; protected long tmpExecutedQueries = 0; - public HttpWorker(String taskID, Integer workerID, Connection connection, Map queries, @Nullable Integer timeLimit, @Nullable Integer timeOut, @Nullable Integer fixedLatency, @Nullable Integer gaussianLatency) { + public HttpWorker(String taskID, Integer workerID, Connection connection, Map queries, @Nullable Integer timeLimit, @Nullable Integer timeOut, @Nullable Integer fixedLatency, @Nullable Integer gaussianLatency) { super(taskID, workerID, connection, queries, timeLimit, timeOut, fixedLatency, gaussianLatency); this.timeoutExecutorPool.setRemoveOnCancelPolicy(true); } diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/impl/MultipleCLIInputWorker.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/impl/MultipleCLIInputWorker.java index 8063276ff..b30e98499 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/impl/MultipleCLIInputWorker.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/impl/MultipleCLIInputWorker.java @@ -42,7 +42,7 @@ public class MultipleCLIInputWorker extends AbstractWorker { protected int numberOfProcesses = 5; private Process currentProcess; - public MultipleCLIInputWorker(String taskID, Integer workerID, Connection connection, Map queries, @Nullable Integer timeLimit, @Nullable Integer timeOut, @Nullable Integer fixedLatency, @Nullable Integer gaussianLatency, String initFinished, String queryFinished, String queryError, @Nullable Integer numberOfProcesses) { + public MultipleCLIInputWorker(String taskID, Integer workerID, Connection connection, Map queries, @Nullable Integer timeLimit, @Nullable Integer timeOut, @Nullable Integer fixedLatency, @Nullable Integer gaussianLatency, String initFinished, String queryFinished, String queryError, @Nullable Integer numberOfProcesses) { super(taskID, workerID, connection, queries, timeLimit, timeOut, fixedLatency, gaussianLatency); this.initFinished = initFinished; this.queryFinished = queryFinished; diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/impl/SPARQLWorker.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/impl/SPARQLWorker.java index 6c6a26e66..a72af5da0 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/impl/SPARQLWorker.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/impl/SPARQLWorker.java @@ -15,8 +15,8 @@ @Shorthand("SPARQLWorker") public class SPARQLWorker extends HttpGetWorker { - public SPARQLWorker(String taskID, Integer workerID, Connection connection, Map queries, @Nullable Integer timeLimit, @Nullable Integer timeOut, @Nullable Integer fixedLatency, @Nullable Integer gaussianLatency, @Nullable String parameterName, @Nullable String responseType) { - super(taskID, workerID, connection, queries, timeLimit, timeOut, fixedLatency, gaussianLatency, parameterName, responseType); - } + public SPARQLWorker(String taskID, Integer workerID, Connection connection, Map queries, @Nullable Integer timeLimit, @Nullable Integer timeOut, @Nullable Integer fixedLatency, @Nullable Integer gaussianLatency, @Nullable String parameterName, @Nullable String responseType) { + super(taskID, workerID, connection, queries, timeLimit, timeOut, fixedLatency, gaussianLatency, parameterName, responseType); + } } diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/impl/UPDATEWorker.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/impl/UPDATEWorker.java index c78b821d2..b6843423a 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/impl/UPDATEWorker.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/impl/UPDATEWorker.java @@ -26,7 +26,7 @@ public class UPDATEWorker extends HttpPostWorker { private int queryCount; - public UPDATEWorker(String taskID, Integer workerID, Connection connection, Map queries, @Nullable Integer timeLimit, @Nullable Integer timeOut, @Nullable Integer fixedLatency, @Nullable Integer gaussianLatency, @Nullable String timerStrategy) { + public UPDATEWorker(String taskID, Integer workerID, Connection connection, Map queries, @Nullable Integer timeLimit, @Nullable Integer timeOut, @Nullable Integer fixedLatency, @Nullable Integer gaussianLatency, @Nullable String timerStrategy) { super(taskID, workerID, connection, queries, timeLimit, timeOut, fixedLatency, gaussianLatency, null, null, "application/sparql-update"); this.timerStrategy = timerStrategy; } diff --git a/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/handler/QueryHandlerTest.java b/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/handler/QueryHandlerTest.java index b7c3e933a..e9ff56084 100644 --- a/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/handler/QueryHandlerTest.java +++ b/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/handler/QueryHandlerTest.java @@ -30,7 +30,7 @@ public class QueryHandlerTest { private final String[] expected; - public QueryHandlerTest(Map config, String[] expected) { + public QueryHandlerTest(Map config, String[] expected) { this.queryHandler = new QueryHandler(config, 0); // workerID 0 results in correct seed for RandomSelector this.expected = expected; } diff --git a/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/tasks/impl/StresstestTest.java b/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/tasks/impl/StresstestTest.java index f065b5dab..13be63266 100644 --- a/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/tasks/impl/StresstestTest.java +++ b/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/tasks/impl/StresstestTest.java @@ -22,9 +22,9 @@ public class StresstestTest { private final String[] queries = new String[]{"a", "b"}; private final String[] queries2 = new String[]{"b", "c"}; - private List> getWorkers(int threads, String[] queries) { - List> workers = new ArrayList<>(); - Map workerConfig = new HashMap<>(); + private List> getWorkers(int threads, String[] queries) { + List> workers = new ArrayList<>(); + Map workerConfig = new HashMap<>(); workerConfig.put("className", MockupWorker.class.getCanonicalName()); workerConfig.put("stringQueries", queries); workerConfig.put("threads", threads); @@ -95,7 +95,7 @@ public void warmupTest() { assertEquals(0.0, end.toEpochMilli() - start.toEpochMilli(), 5.0); //check if executing - Map warmup = new LinkedHashMap<>(); + Map warmup = new LinkedHashMap<>(); warmup.put("workers", getWorkers(2, this.queries)); warmup.put("timeLimit", 350); @@ -114,7 +114,7 @@ public void warmupTest() { @Test public void workerCreationTest() { - List> worker = getWorkers(2, this.queries); + List> worker = getWorkers(2, this.queries); worker.addAll(getWorkers(1, this.queries2)); Stresstest task = new Stresstest(5000, worker); diff --git a/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/worker/HTTPWorkerTest.java b/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/worker/HTTPWorkerTest.java index b9329d8ad..469fa9825 100644 --- a/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/worker/HTTPWorkerTest.java +++ b/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/worker/HTTPWorkerTest.java @@ -31,7 +31,7 @@ public class HTTPWorkerTest { private static SocketConnection fastConnection; private final String service; private final Boolean isPost; - private final HashMap queries; + private final HashMap queries; private final String queriesFile = "src/test/resources/workers/single-query.txt"; private final String responseType; diff --git a/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/worker/MockupWorker.java b/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/worker/MockupWorker.java index 08c3e0476..38c756d7b 100644 --- a/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/worker/MockupWorker.java +++ b/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/worker/MockupWorker.java @@ -21,8 +21,8 @@ public String[] getStringQueries() { return queries; } - private static Map getQueryConfig() { - Map queryConfig = new HashMap<>(); + private static Map getQueryConfig() { + Map queryConfig = new HashMap<>(); queryConfig.put("location", "src/test/resources/mockupq.txt"); return queryConfig; } diff --git a/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/worker/UPDATEWorkerTest.java b/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/worker/UPDATEWorkerTest.java index b5c770944..3a0258bbe 100644 --- a/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/worker/UPDATEWorkerTest.java +++ b/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/worker/UPDATEWorkerTest.java @@ -31,17 +31,17 @@ public class UPDATEWorkerTest { private static SocketConnection fastConnection; private final String service; private final String timerStrategy; - private final Map queriesFile; + private final Map queriesFile; private final int expectedExec; private String outputDir; - public UPDATEWorkerTest(String timerStrategy, Map queriesFile, int expectedExec) { + public UPDATEWorkerTest(String timerStrategy, Map queriesFile, int expectedExec) { this.service = "http://localhost:8025/test"; this.timerStrategy = timerStrategy; this.queriesFile = queriesFile; this.expectedExec = expectedExec; //warmup - Map warmupQueries = new HashMap<>(); + Map warmupQueries = new HashMap<>(); warmupQueries.put("location", "src/test/resources/workers/single-query.txt"); SPARQLWorker worker = new SPARQLWorker("", 1, getConnection(), warmupQueries, null, null, null, null, null, null); worker.executeQuery("INSERT DATA {", "1"); @@ -53,30 +53,30 @@ public UPDATEWorkerTest(String timerStrategy, Map queriesFile, i public static Collection data() { Collection testData = new ArrayList<>(); - Map queries0 = new HashMap<>(); + Map queries0 = new HashMap<>(); queries0.put("location", "src/test/resources/workers/updates"); queries0.put("format", "folder"); testData.add(new Object[]{"none", queries0, 4}); - Map queries1 = new HashMap<>(); + Map queries1 = new HashMap<>(); queries1.put("location", "src/test/resources/workers/updates"); queries1.put("format", "folder"); testData.add(new Object[]{"fixed", queries1, 4}); - Map queries2 = new HashMap<>(); + Map queries2 = new HashMap<>(); queries2.put("location", "src/test/resources/workers/updates"); queries2.put("format", "folder"); testData.add(new Object[]{"distributed", queries2, 4}); - Map queries3 = new HashMap<>(); + Map queries3 = new HashMap<>(); queries3.put("location", "src/test/resources/workers/updates.txt"); testData.add(new Object[]{"none", queries3, 3}); - Map queries4 = new HashMap<>(); + Map queries4 = new HashMap<>(); queries4.put("location", "src/test/resources/workers/updates.txt"); testData.add(new Object[]{"fixed", queries4, 3}); - Map queries5 = new HashMap<>(); + Map queries5 = new HashMap<>(); queries5.put("location", "src/test/resources/workers/updates.txt"); testData.add(new Object[]{"distributed", queries5, 3}); return testData; diff --git a/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/worker/impl/CLIWorkersTests.java b/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/worker/impl/CLIWorkersTests.java index 8ca69f5e4..faf5d03a4 100644 --- a/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/worker/impl/CLIWorkersTests.java +++ b/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/worker/impl/CLIWorkersTests.java @@ -161,8 +161,8 @@ public void checkCLI() throws IOException { assertEquals(2L, p.get(COMMON.RECEIVE_DATA_SIZE)); } - private Map getQueryConfig() { - Map config = new HashMap<>(); + private Map getQueryConfig() { + Map config = new HashMap<>(); config.put("location", "src/test/resources/updates/empty.nt"); return config; } diff --git a/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/worker/impl/HttpPostWorkerTest.java b/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/worker/impl/HttpPostWorkerTest.java index d2e5d904b..4fff0e5b7 100644 --- a/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/worker/impl/HttpPostWorkerTest.java +++ b/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/worker/impl/HttpPostWorkerTest.java @@ -16,8 +16,8 @@ public class HttpPostWorkerTest { - private static Map getDefaultQueryConfig() { - Map queries = new HashMap<>(); + private static Map getDefaultQueryConfig() { + Map queries = new HashMap<>(); queries.put("location", "src/test/resources/workers/single-query.txt"); return queries; } From f6281fb7df6663d97b3bf6da2c541a02db4fb535 Mon Sep 17 00:00:00 2001 From: nck-mlcnv <32801560+nck-mlcnv@users.noreply.github.com> Date: Wed, 8 Mar 2023 11:51:16 +0100 Subject: [PATCH 29/47] reformat schema-file --- schema/iguana-schema.json | 460 +++++++++++++++----------------------- 1 file changed, 186 insertions(+), 274 deletions(-) diff --git a/schema/iguana-schema.json b/schema/iguana-schema.json index 842e4b363..cfa2f951c 100644 --- a/schema/iguana-schema.json +++ b/schema/iguana-schema.json @@ -5,311 +5,204 @@ "connection": { "type": "object", "properties": { - "endpoint": { "type": "string" }, - "updateEndpoint": { "type": "string" }, - "user": { "type": "string" }, - "password": { "type": "string" }, - "version": { - "type": "string" - } + "endpoint": {"type": "string"}, + "updateEndpoint": {"type": "string"}, + "user": {"type": "string"}, + "password": {"type": "string"}, + "version": {"type": "string"} }, "required": ["endpoint"] }, - "warmup" : { + + "warmup": { "type": "object", "properties": { - "timeLimit": { - "type": "integer" - }, - "queryHandler": { - "$ref": "#/definitions/genericClassObject" - }, + "timeLimit": {"type": "integer"}, + "queryHandler": {"$ref": "#/definitions/genericClassObject"}, "workers": { "type": "array", "items": { - "oneOf": [ - { - "$ref": "#/definitions/AbstractWorker" - } - ] + "oneOf": [{"$ref": "#/definitions/AbstractWorker"}] } } }, - "required": ["workers","timeLimit"] - }, + "required": ["workers", "timeLimit"] + }, + "stresstest": { "type": "object", "properties": { - "timeLimit": { "type": "integer" }, + "timeLimit": {"type": "integer"}, "noOfQueryMixes": {"type": "integer"}, - "queryHandler": {"$ref" : "#/definitions/genericClassObject" }, - "warmup" : {"$ref" : "#/definitions/warmup"}, + "queryHandler": {"$ref": "#/definitions/genericClassObject"}, + "warmup": {"$ref": "#/definitions/warmup"}, "workers": { "type": "array", "items": { - "oneOf": [ - { - "$ref": "#/definitions/AbstractWorker" - } - ] + "oneOf": [{"$ref": "#/definitions/AbstractWorker"}] } } }, "required": ["queryHandler", "workers"] }, + "AbstractWorker": { "type": "object", "properties": { - "className": { - "type": "string" - } - - }, - "allOf": [{ - "if": { - "properties": { - "className" : { - "oneOf": [ {"const": "SPARQLWorker"},{"const": "org.aksw.iguana.cc.worker.impl.SPARQLWorker"}] - } - } + "className": {"type": "string"} }, - "then": + "allOf": [ { - "additionalProperties": {"type": "undefined"}, - - "required": [ - "className", - "threads", - "queriesFile" - ], - "properties": { - "className": { - "type": "string" - }, - "threads": { - "type": "integer" - }, - "queriesFile": { - "type": "string" - }, - "timeOut": { - "type": "integer" - }, - "fixedLatency": { - "type": "integer" - }, - "gaussianLatency": { - "type": "integer" - }, - "responseType": { - "type": "string" - }, - "parameterName": { - "type": "string" + "if": { + "properties": { + "className" : { + "oneOf": [ {"const": "SPARQLWorker"},{"const": "org.aksw.iguana.cc.worker.impl.SPARQLWorker"}] + } } + }, + "then": { + "properties": { + "className": {"type": "string"}, + "threads": {"type": "integer"}, + "queriesFile": {"type": "string"}, + "timeOut": {"type": "integer"}, + "fixedLatency": {"type": "integer"}, + "gaussianLatency": {"type": "integer"}, + "responseType": {"type": "string"}, + "parameterName": {"type": "string"} + }, + "additionalProperties": {"type": "undefined"}, + "required": ["className", "threads", "queriesFile"] } - } - - }, + }, { - "if": { - "properties": { - "className" : { - "oneOf": [{"const": "UPDATEWorker"},{"const": "org.aksw.iguana.cc.worker.impl.UPDATEWorker"}] - } - } - }, - "then": - {"required": ["className", "threads", "queriesFile"], + "if": { "properties": { - "className": { - "type": "string" - }, - "threads" : {"type": "integer"}, - "queriesFile" : {"type": "string"}, - "timeOut" : {"type": "integer"}, - "fixedLatency" : {"type": "integer"}, - "gaussianLatency" : {"type": "integer"}, - "timerStrategy" : {"type": "string"} + "className" : { + "oneOf": [{"const": "UPDATEWorker"},{"const": "org.aksw.iguana.cc.worker.impl.UPDATEWorker"}] + } + } }, - "additionalProperties": {"type": "undefined"} - } - - }, - {"if": {"properties": { - "className" : { - "oneOf": [{"const": "MultipleCLIInputWorker"}, {"const": "org.aksw.iguana.cc.worker.impl.MultipleCLIInputWorker"}] + "then": { + "properties": { + "className": {"type": "string"}, + "threads": {"type": "integer"}, + "queriesFile": {"type": "string"}, + "timeOut": {"type": "integer"}, + "fixedLatency": {"type": "integer"}, + "gaussianLatency": {"type": "integer"}, + "timerStrategy": {"type": "string"} + }, + "additionalProperties": {"type": "undefined"}, + "required": ["className", "threads", "queriesFile"] } - }}, - "then": - {"required": ["className", "threads", "queriesFile", "queryError", "queryFinished", "initFinished"], - "properties": { - "className": { - "type": "string" - }, - "threads" : {"type": "integer"}, - "queriesFile" : {"type": "string"}, - "timeOut" : {"type": "integer"}, - "fixedLatency" : {"type": "integer"}, - "gaussianLatency" : {"type": "integer"}, - "queryError" : {"type": "string"}, - "queryFinished" : {"type": "string"}, - "initFinished" : {"type": "string"}, - "numberOfProcesses" : {"type": "integer"} - }, "additionalProperties": {"type": "undefined"} - } }, { - "if": { - "properties": { - "className" : { - "oneOf": [{"const": "CLIInputWorker"}, {"const": "org.aksw.iguana.cc.worker.impl.CLIInputWorker"}] - } - } - }, - "then": - {"required": ["className", "threads", "queriesFile", "queryError", "queryFinished", "initFinished"], + "if": { "properties": { - "className": { - "type": "string" + "className" : { + "oneOf": [{"const": "MultipleCLIInputWorker"}, {"const": "org.aksw.iguana.cc.worker.impl.MultipleCLIInputWorker"}] + } + }}, + "then": { + "properties": { + "className": {"type": "string"}, + "threads": {"type": "integer"}, + "queriesFile": {"type": "string"}, + "timeOut": {"type": "integer"}, + "fixedLatency": {"type": "integer"}, + "gaussianLatency": {"type": "integer"}, + "queryError": {"type": "string"}, + "queryFinished": {"type": "string"}, + "initFinished": {"type": "string"}, + "numberOfProcesses": {"type": "integer"} }, - "threads" : {"type": "integer"}, - "queriesFile" : {"type": "string"}, - "timeOut" : {"type": "integer"}, - "fixedLatency" : {"type": "integer"}, - "gaussianLatency" : {"type": "integer"}, - "queryError" : {"type": "string"}, - "queryFinished" : {"type": "string"}, - "initFinished" : {"type": "string"} - }, "additionalProperties": {"type": "undefined"} + "additionalProperties": {"type": "undefined"}, + "required": ["className", "threads", "queriesFile", "queryError", "queryFinished", "initFinished"] } - }, + }, { - "if": { - "properties": { - "className" : { - "oneOf": [{"const": "CLIPrefixWorker"}, {"const": "org.aksw.iguana.cc.worker.impl.CLIPrefixWorker"}] - } - } - }, - "then": { - "required": [ - "className", - "threads", - "queriesFile", - "queryError", - "queryFinished", - "initFinished", - "queryPrefix", - "querySuffix" - ], - "properties": { - "className": { - "type": "string" - }, - "threads": { - "type": "integer" - }, - "queriesFile": { - "type": "string" - }, - "timeOut": { - "type": "integer" - }, - "fixedLatency": { - "type": "integer" - }, - "gaussianLatency": { - "type": "integer" - }, - "numberOfProcesses": { - "type": "integer" - }, - "queryError": { - "type": "string" - }, - "queryFinished": { - "type": "string" - }, - "initFinished": { - "type": "string" - }, - "querySuffix": { - "type": "string" + "if": { + "properties": { + "className" : { + "oneOf": [{"const": "CLIInputWorker"}, {"const": "org.aksw.iguana.cc.worker.impl.CLIInputWorker"}] + } + } }, - "queryPrefix": { - "type": "string" + "then": { + "properties": { + "className": {"type": "string"}, + "threads": {"type": "integer"}, + "queriesFile": {"type": "string"}, + "timeOut": {"type": "integer"}, + "fixedLatency": {"type": "integer"}, + "gaussianLatency": {"type": "integer"}, + "queryError": {"type": "string"}, + "queryFinished": {"type": "string"}, + "initFinished": {"type": "string"} + }, + "additionalProperties": {"type": "undefined"}, + "required": ["className", "threads", "queriesFile", "queryError", "queryFinished", "initFinished"] } }, - "additionalProperties": {"type": "undefined"} - } - - }, - {"if": { - "properties": { - "className" : { - "oneOf": [{"const": "MultipleCLIInputFileWorker"}, {"const": "org.aksw.iguana.cc.worker.impl.MultipleCLIInputFileWorker"}] + { + "if": { + "properties": { + "className": { + "oneOf": [{"const": "CLIPrefixWorker"}, {"const": "org.aksw.iguana.cc.worker.impl.CLIPrefixWorker"}] + } } - } - }, + }, "then": { + "properties": { + "className": {"type": "string"}, + "threads": {"type": "integer"}, + "queriesFile": {"type": "string"}, + "timeOut": {"type": "integer"}, + "fixedLatency": {"type": "integer"}, + "gaussianLatency": {"type": "integer"}, + "numberOfProcesses": {"type": "integer"}, + "queryError": {"type": "string"}, + "queryFinished": {"type": "string"}, + "initFinished": {"type": "string"}, + "querySuffix": {"type": "string"}, + "queryPrefix": {"type": "string"} + }, + "additionalProperties": {"type": "undefined"}, "required": [ "className", "threads", "queriesFile", - "directory", "queryError", "queryFinished", - "initFinished" - ], - "properties": { - "className": { - "type": "string" - }, - "threads": { - "type": "integer" - }, - "queriesFile": { - "type": "string" - }, - "timeOut": { - "type": "integer" - }, - "fixedLatency": { - "type": "integer" - }, - "gaussianLatency": { - "type": "integer" - }, - "queryError": { - "type": "string" - }, - "queryFinished": { - "type": "string" - }, - "initFinished": { - "type": "string" - }, - "directory": { - "type": "string" - }, - "numberOfProcesses": { - "type": "integer" - } - }, - "additionalProperties": {"type": "undefined"} + "initFinished", + "queryPrefix", + "querySuffix"] } }, { "if": { - "properties": { - "className": { - "oneOf": [{"const": "CLIInputFileWorker"}, {"const": "org.aksw.iguana.cc.worker.impl.CLIInputFileWorker"}] + "properties": { + "className": { + "oneOf": [{"const": "MultipleCLIInputFileWorker"}, {"const": "org.aksw.iguana.cc.worker.impl.MultipleCLIInputFileWorker"}] } } }, "then": { - "allOf": [{ + "properties": { + "className": {"type": "string"}, + "threads": {"type": "integer"}, + "queriesFile": {"type": "string"}, + "timeOut": {"type": "integer"}, + "fixedLatency": {"type": "integer"}, + "gaussianLatency": {"type": "integer"}, + "queryError": {"type": "string"}, + "queryFinished": {"type": "string"}, + "initFinished": {"type": "string"}, + "directory": {"type": "string"}, + "numberOfProcesses": {"type": "integer"} + }, + "additionalProperties": {"type": "undefined"}, "required": [ "className", "threads", @@ -318,52 +211,75 @@ "queryError", "queryFinished", "initFinished" - ]}, - {"properties": { - "className": { - "type": "string" + ] + } + }, + { + "if": { + "properties": { + "className": { + "oneOf": [{"const": "CLIInputFileWorker"}, {"const": "org.aksw.iguana.cc.worker.impl.CLIInputFileWorker"}] + } + } + }, + "then": { + "allOf": [ + { + "required": [ + "className", + "threads", + "queriesFile", + "directory", + "queryError", + "queryFinished", + "initFinished" + ] + }, + { + "properties": { + "className": {"type": "string"}, + "threads": {"type": "integer"}, + "queriesFile": {"type": "string"}, + "timeOut": {"type": "integer"}, + "fixedLatency": {"type": "integer"}, + "gaussianLatency": {"type": "integer"}, + "queryError": {"type": "string"}, + "queryFinished": {"type": "string"}, + "initFinished": {"type": "string"}, + "directory": {"type": "string"} }, - "threads" : {"type": "integer"}, - "queriesFile" : {"type": "string"}, - "timeOut" : {"type": "integer"}, - "fixedLatency" : {"type": "integer"}, - "gaussianLatency" : {"type": "integer"}, - "queryError" : {"type": "string"}, - "queryFinished" : {"type": "string"}, - "initFinished" : {"type": "string"}, - "directory" : {"type" : "string"} - }, "additionalProperties": {"type": "undefined"} - }] + "additionalProperties": {"type": "undefined"} + } + ] } } - ] + ] }, + "task": { "type": "object", "properties": { - "className": { "type": "string" }, - "configuration": { + "className": {"type": "string"}, + "configuration": { "oneOf": [{"$ref": "#/definitions/stresstest"}] } }, "required": ["className", "configuration"] }, + "genericClassObject": { "type": "object", "properties": { - "className": { "type": "string" }, + "className": {"type": "string"}, "configuration": { "type": "object" } }, "required": ["className"] - } - }, "type": "object", - "properties": { "connections": { "type": "array", @@ -387,12 +303,8 @@ "$ref":"#/definitions/task" } }, - "preScriptHook": { - "type": "string" - }, - "postScriptHook": { - "type": "string" - }, + "preScriptHook": {"type": "string"}, + "postScriptHook": {"type": "string"}, "metrics": { "type": "array", "items": { From b4ce0f11c3269772324e61381ddaec3cac080514 Mon Sep 17 00:00:00 2001 From: nck-mlcnv <32801560+nck-mlcnv@users.noreply.github.com> Date: Wed, 8 Mar 2023 12:47:43 +0100 Subject: [PATCH 30/47] update configuration file schema --- .../src/main/resources/iguana-schema.json | 502 ++++++++---------- schema/iguana-schema.json | 123 +++-- 2 files changed, 315 insertions(+), 310 deletions(-) diff --git a/iguana.corecontroller/src/main/resources/iguana-schema.json b/iguana.corecontroller/src/main/resources/iguana-schema.json index f215da2c4..4374a3c5b 100644 --- a/iguana.corecontroller/src/main/resources/iguana-schema.json +++ b/iguana.corecontroller/src/main/resources/iguana-schema.json @@ -5,362 +5,326 @@ "connection": { "type": "object", "properties": { - "endpoint": { "type": "string" }, - "updateEndpoint": { "type": "string" }, - "user": { "type": "string" }, - "password": { "type": "string" } + "endpoint": {"type": "string"}, + "updateEndpoint": {"type": "string"}, + "user": {"type": "string"}, + "password": {"type": "string"}, + "version": {"type": "string"} }, "required": ["endpoint"] }, - "warmup" : { + + "queries": { "type": "object", "properties": { - "timeLimit": { - "type": "integer" + "location": {"type": "string"}, + "format": { + "oneOf": [ + {"type": "string"}, + { + "type": "object", + "properties": { + "separator": {"type": "string"} + } + } + ] + }, + "caching": {"type": "boolean"}, + "order": { + "oneOf": [ + {"type": "string"}, + { + "type": "object", + "properties": { + "random": { + "type": "object", + "properties": { + "seed": {"type": "integer"} + }, + "required": ["seed"] + } + } + } + ] }, - "queryHandler": { - "$ref": "#/definitions/genericClassObject" + "pattern": { + "type": "object", + "properties": { + "endpoint": {"type": "string"}, + "outputFolder": {"type": "string"}, + "limit": {"type": "integer"} + } }, + "lang": {"type": "string"} + }, + "required": ["location"] + }, + + "warmup": { + "type": "object", + "properties": { + "timeLimit": {"type": "integer"}, "workers": { "type": "array", "items": { - "oneOf": [ - { - "$ref": "#/definitions/AbstractWorker" - } - ] + "oneOf": [{"$ref": "#/definitions/AbstractWorker"}] } } }, - "required": ["workers","timeLimit"] - }, + "required": ["workers", "timeLimit"] + }, + "stresstest": { "type": "object", "properties": { - "timeLimit": { "type": "integer" }, + "timeLimit": {"type": "integer"}, "noOfQueryMixes": {"type": "integer"}, - "queryHandler": {"$ref" : "#/definitions/genericClassObject" }, - "warmup" : {"$ref" : "#/definitions/warmup"}, + "warmup": {"$ref": "#/definitions/warmup"}, "workers": { "type": "array", "items": { - "oneOf": [ - { - "$ref": "#/definitions/AbstractWorker" - } - ] + "oneOf": [{"$ref": "#/definitions/AbstractWorker"}] } } }, - "required": ["queryHandler", "workers"] + "required": ["workers"] }, + "AbstractWorker": { "type": "object", "properties": { - "className": { - "type": "string" - } - - }, - "allOf": [{ - "if": { - "properties": { - "className" : { - "oneOf": [ {"const": "SPARQLWorker"},{"const": "org.aksw.iguana.cc.worker.impl.SPARQLWorker"}] - } - } + "className": {"type": "string"} }, - "then": + "allOf": [ { - "additionalProperties": {"type": "undefined"}, - - "required": [ - "className", - "threads", - "queriesFile" - ], - "properties": { - "className": { - "type": "string" - }, - "threads": { - "type": "integer" - }, - "queriesFile": { - "type": "string" - }, - "timeOut": { - "type": "integer" - }, - "fixedLatency": { - "type": "integer" - }, - "gaussianLatency": { - "type": "integer" - }, - "responseType": { - "type": "string" - }, - "parameterName": { - "type": "string" + "if": { + "properties": { + "className" : { + "oneOf": [ {"const": "SPARQLWorker"},{"const": "org.aksw.iguana.cc.worker.impl.SPARQLWorker"}] + } } + }, + "then": { + "properties": { + "className": {"type": "string"}, + "threads": {"type": "integer"}, + "timeOut": {"type": "integer"}, + "fixedLatency": {"type": "integer"}, + "gaussianLatency": {"type": "integer"}, + "responseType": {"type": "string"}, + "parameterName": {"type": "string"}, + "queries": {"$ref": "#/definitions/queries"} + }, + "additionalProperties": {"type": "undefined"}, + "required": ["className", "threads", "queries"] } - } - - }, + }, { - "if": { - "properties": { - "className" : { - "oneOf": [{"const": "UPDATEWorker"},{"const": "org.aksw.iguana.cc.worker.impl.UPDATEWorker"}] - } - } - }, - "then": - {"required": ["className", "threads", "queriesFile"], + "if": { "properties": { - "className": { - "type": "string" - }, - "threads" : {"type": "integer"}, - "queriesFile" : {"type": "string"}, - "timeOut" : {"type": "integer"}, - "fixedLatency" : {"type": "integer"}, - "gaussianLatency" : {"type": "integer"}, - "timerStrategy" : {"type": "string"} + "className" : { + "oneOf": [{"const": "UPDATEWorker"},{"const": "org.aksw.iguana.cc.worker.impl.UPDATEWorker"}] + } + } }, - "additionalProperties": {"type": "undefined"} - } - - }, - {"if": {"properties": { - "className" : { - "oneOf": [{"const": "MultipleCLIInputWorker"}, {"const": "org.aksw.iguana.cc.worker.impl.MultipleCLIInputWorker"}] + "then": { + "properties": { + "className": {"type": "string"}, + "threads": {"type": "integer"}, + "timeOut": {"type": "integer"}, + "fixedLatency": {"type": "integer"}, + "gaussianLatency": {"type": "integer"}, + "timerStrategy": {"type": "string"}, + "queries": {"$ref": "#/definitions/queries"} + }, + "additionalProperties": {"type": "undefined"}, + "required": ["className", "threads", "queries"] } - }}, - "then": - {"required": ["className", "threads", "queriesFile", "queryError", "queryFinished", "initFinished"], - "properties": { - "className": { - "type": "string" - }, - "threads" : {"type": "integer"}, - "queriesFile" : {"type": "string"}, - "timeOut" : {"type": "integer"}, - "fixedLatency" : {"type": "integer"}, - "gaussianLatency" : {"type": "integer"}, - "queryError" : {"type": "string"}, - "queryFinished" : {"type": "string"}, - "initFinished" : {"type": "string"}, - "numberOfProcesses" : {"type": "integer"} - }, "additionalProperties": {"type": "undefined"} - } }, { - "if": { - "properties": { - "className" : { - "oneOf": [{"const": "CLIInputWorker"}, {"const": "org.aksw.iguana.cc.worker.impl.CLIInputWorker"}] - } - } - }, - "then": - {"required": ["className", "threads", "queriesFile", "queryError", "queryFinished", "initFinished"], + "if": { + "properties": { + "className" : { + "oneOf": [{"const": "MultipleCLIInputWorker"}, {"const": "org.aksw.iguana.cc.worker.impl.MultipleCLIInputWorker"}] + } + }}, + "then": { "properties": { - "className": { - "type": "string" + "className": {"type": "string"}, + "threads": {"type": "integer"}, + "timeOut": {"type": "integer"}, + "fixedLatency": {"type": "integer"}, + "gaussianLatency": {"type": "integer"}, + "queryError": {"type": "string"}, + "queryFinished": {"type": "string"}, + "initFinished": {"type": "string"}, + "numberOfProcesses": {"type": "integer"}, + "queries": {"$ref": "#/definitions/queries"} }, - "threads" : {"type": "integer"}, - "queriesFile" : {"type": "string"}, - "timeOut" : {"type": "integer"}, - "fixedLatency" : {"type": "integer"}, - "gaussianLatency" : {"type": "integer"}, - "queryError" : {"type": "string"}, - "queryFinished" : {"type": "string"}, - "initFinished" : {"type": "string"} - }, "additionalProperties": {"type": "undefined"} + "additionalProperties": {"type": "undefined"}, + "required": ["className", "threads", "queryError", "queryFinished", "initFinished", "queries"] } - }, + }, { - "if": { - "properties": { - "className" : { - "oneOf": [{"const": "CLIPrefixWorker"}, {"const": "org.aksw.iguana.cc.worker.impl.CLIPrefixWorker"}] - } - } - }, - "then": { - "required": [ - "className", - "threads", - "queriesFile", - "queryError", - "queryFinished", - "initFinished", - "queryPrefix", - "querySuffix" - ], - "properties": { - "className": { - "type": "string" - }, - "threads": { - "type": "integer" - }, - "queriesFile": { - "type": "string" - }, - "timeOut": { - "type": "integer" - }, - "fixedLatency": { - "type": "integer" - }, - "gaussianLatency": { - "type": "integer" - }, - "numberOfProcesses": { - "type": "integer" - }, - "queryError": { - "type": "string" - }, - "queryFinished": { - "type": "string" - }, - "initFinished": { - "type": "string" - }, - "querySuffix": { - "type": "string" + "if": { + "properties": { + "className" : { + "oneOf": [{"const": "CLIInputWorker"}, {"const": "org.aksw.iguana.cc.worker.impl.CLIInputWorker"}] + } + } }, - "queryPrefix": { - "type": "string" + "then": { + "properties": { + "className": {"type": "string"}, + "threads": {"type": "integer"}, + "timeOut": {"type": "integer"}, + "fixedLatency": {"type": "integer"}, + "gaussianLatency": {"type": "integer"}, + "queryError": {"type": "string"}, + "queryFinished": {"type": "string"}, + "initFinished": {"type": "string"}, + "queries": {"$ref": "#/definitions/queries"} + }, + "additionalProperties": {"type": "undefined"}, + "required": ["className", "threads", "queryError", "queryFinished", "initFinished", "queries"] } }, - "additionalProperties": {"type": "undefined"} - } - - }, - {"if": { - "properties": { - "className" : { - "oneOf": [{"const": "MultipleCLIInputFileWorker"}, {"const": "org.aksw.iguana.cc.worker.impl.MultipleCLIInputFileWorker"}] + { + "if": { + "properties": { + "className": { + "oneOf": [{"const": "CLIPrefixWorker"}, {"const": "org.aksw.iguana.cc.worker.impl.CLIPrefixWorker"}] + } } - } - }, + }, "then": { + "properties": { + "className": {"type": "string"}, + "threads": {"type": "integer"}, + "timeOut": {"type": "integer"}, + "fixedLatency": {"type": "integer"}, + "gaussianLatency": {"type": "integer"}, + "numberOfProcesses": {"type": "integer"}, + "queryError": {"type": "string"}, + "queryFinished": {"type": "string"}, + "initFinished": {"type": "string"}, + "querySuffix": {"type": "string"}, + "queryPrefix": {"type": "string"}, + "queries": {"$ref": "#/definitions/queries"} + }, + "additionalProperties": {"type": "undefined"}, "required": [ "className", "threads", - "queriesFile", - "directory", "queryError", "queryFinished", - "initFinished" - ], - "properties": { - "className": { - "type": "string" - }, - "threads": { - "type": "integer" - }, - "queriesFile": { - "type": "string" - }, - "timeOut": { - "type": "integer" - }, - "fixedLatency": { - "type": "integer" - }, - "gaussianLatency": { - "type": "integer" - }, - "queryError": { - "type": "string" - }, - "queryFinished": { - "type": "string" - }, - "initFinished": { - "type": "string" - }, - "directory": { - "type": "string" - }, - "numberOfProcesses": { - "type": "integer" - } - }, - "additionalProperties": {"type": "undefined"} + "initFinished", + "queryPrefix", + "querySuffix", + "queries" + ] } }, { "if": { "properties": { "className": { - "oneOf": [{"const": "CLIInputFileWorker"}, {"const": "org.aksw.iguana.cc.worker.impl.CLIInputFileWorker"}] + "oneOf": [{"const": "MultipleCLIInputFileWorker"}, {"const": "org.aksw.iguana.cc.worker.impl.MultipleCLIInputFileWorker"}] } } }, "then": { - "allOf": [{ + "properties": { + "className": {"type": "string"}, + "threads": {"type": "integer"}, + "timeOut": {"type": "integer"}, + "fixedLatency": {"type": "integer"}, + "gaussianLatency": {"type": "integer"}, + "queryError": {"type": "string"}, + "queryFinished": {"type": "string"}, + "initFinished": {"type": "string"}, + "directory": {"type": "string"}, + "numberOfProcesses": {"type": "integer"}, + "queries": {"$ref": "#/definitions/queries"} + }, + "additionalProperties": {"type": "undefined"}, "required": [ "className", "threads", - "queriesFile", "directory", "queryError", "queryFinished", - "initFinished" - ]}, - {"properties": { - "className": { - "type": "string" + "initFinished", + "queries" + ] + } + }, + { + "if": { + "properties": { + "className": { + "oneOf": [{"const": "CLIInputFileWorker"}, {"const": "org.aksw.iguana.cc.worker.impl.CLIInputFileWorker"}] + } + } + }, + "then": { + "allOf": [ + { + "properties": { + "className": {"type": "string"}, + "threads": {"type": "integer"}, + "timeOut": {"type": "integer"}, + "fixedLatency": {"type": "integer"}, + "gaussianLatency": {"type": "integer"}, + "queryError": {"type": "string"}, + "queryFinished": {"type": "string"}, + "initFinished": {"type": "string"}, + "directory": {"type": "string"}, + "queries": {"$ref": "#/definitions/queries"} }, - "threads" : {"type": "integer"}, - "queriesFile" : {"type": "string"}, - "timeOut" : {"type": "integer"}, - "fixedLatency" : {"type": "integer"}, - "gaussianLatency" : {"type": "integer"}, - "queryError" : {"type": "string"}, - "queryFinished" : {"type": "string"}, - "initFinished" : {"type": "string"}, - "directory" : {"type" : "string"} - }, "additionalProperties": {"type": "undefined"} - }] + "additionalProperties": {"type": "undefined"} + }, + { + "required": [ + "className", + "threads", + "directory", + "queryError", + "queryFinished", + "initFinished", + "queries" + ] + } + ] } } - ] + ] }, + "task": { "type": "object", "properties": { - "className": { "type": "string" }, - "configuration": { + "className": {"type": "string"}, + "configuration": { "oneOf": [{"$ref": "#/definitions/stresstest"}] } }, "required": ["className", "configuration"] }, + "genericClassObject": { "type": "object", "properties": { - "className": { "type": "string" }, + "className": {"type": "string"}, "configuration": { "type": "object" } }, "required": ["className"] - } - }, "type": "object", - "properties": { "connections": { "type": "array", @@ -384,12 +348,8 @@ "$ref":"#/definitions/task" } }, - "preScriptHook": { - "type": "string" - }, - "postScriptHook": { - "type": "string" - }, + "preScriptHook": {"type": "string"}, + "postScriptHook": {"type": "string"}, "metrics": { "type": "array", "items": { diff --git a/schema/iguana-schema.json b/schema/iguana-schema.json index cfa2f951c..4374a3c5b 100644 --- a/schema/iguana-schema.json +++ b/schema/iguana-schema.json @@ -14,11 +14,56 @@ "required": ["endpoint"] }, + "queries": { + "type": "object", + "properties": { + "location": {"type": "string"}, + "format": { + "oneOf": [ + {"type": "string"}, + { + "type": "object", + "properties": { + "separator": {"type": "string"} + } + } + ] + }, + "caching": {"type": "boolean"}, + "order": { + "oneOf": [ + {"type": "string"}, + { + "type": "object", + "properties": { + "random": { + "type": "object", + "properties": { + "seed": {"type": "integer"} + }, + "required": ["seed"] + } + } + } + ] + }, + "pattern": { + "type": "object", + "properties": { + "endpoint": {"type": "string"}, + "outputFolder": {"type": "string"}, + "limit": {"type": "integer"} + } + }, + "lang": {"type": "string"} + }, + "required": ["location"] + }, + "warmup": { "type": "object", "properties": { "timeLimit": {"type": "integer"}, - "queryHandler": {"$ref": "#/definitions/genericClassObject"}, "workers": { "type": "array", "items": { @@ -34,7 +79,6 @@ "properties": { "timeLimit": {"type": "integer"}, "noOfQueryMixes": {"type": "integer"}, - "queryHandler": {"$ref": "#/definitions/genericClassObject"}, "warmup": {"$ref": "#/definitions/warmup"}, "workers": { "type": "array", @@ -43,7 +87,7 @@ } } }, - "required": ["queryHandler", "workers"] + "required": ["workers"] }, "AbstractWorker": { @@ -64,15 +108,15 @@ "properties": { "className": {"type": "string"}, "threads": {"type": "integer"}, - "queriesFile": {"type": "string"}, "timeOut": {"type": "integer"}, "fixedLatency": {"type": "integer"}, "gaussianLatency": {"type": "integer"}, "responseType": {"type": "string"}, - "parameterName": {"type": "string"} + "parameterName": {"type": "string"}, + "queries": {"$ref": "#/definitions/queries"} }, "additionalProperties": {"type": "undefined"}, - "required": ["className", "threads", "queriesFile"] + "required": ["className", "threads", "queries"] } }, { @@ -87,14 +131,14 @@ "properties": { "className": {"type": "string"}, "threads": {"type": "integer"}, - "queriesFile": {"type": "string"}, "timeOut": {"type": "integer"}, "fixedLatency": {"type": "integer"}, "gaussianLatency": {"type": "integer"}, - "timerStrategy": {"type": "string"} + "timerStrategy": {"type": "string"}, + "queries": {"$ref": "#/definitions/queries"} }, "additionalProperties": {"type": "undefined"}, - "required": ["className", "threads", "queriesFile"] + "required": ["className", "threads", "queries"] } }, { @@ -108,17 +152,17 @@ "properties": { "className": {"type": "string"}, "threads": {"type": "integer"}, - "queriesFile": {"type": "string"}, "timeOut": {"type": "integer"}, "fixedLatency": {"type": "integer"}, "gaussianLatency": {"type": "integer"}, "queryError": {"type": "string"}, "queryFinished": {"type": "string"}, "initFinished": {"type": "string"}, - "numberOfProcesses": {"type": "integer"} + "numberOfProcesses": {"type": "integer"}, + "queries": {"$ref": "#/definitions/queries"} }, "additionalProperties": {"type": "undefined"}, - "required": ["className", "threads", "queriesFile", "queryError", "queryFinished", "initFinished"] + "required": ["className", "threads", "queryError", "queryFinished", "initFinished", "queries"] } }, { @@ -133,16 +177,16 @@ "properties": { "className": {"type": "string"}, "threads": {"type": "integer"}, - "queriesFile": {"type": "string"}, "timeOut": {"type": "integer"}, "fixedLatency": {"type": "integer"}, "gaussianLatency": {"type": "integer"}, "queryError": {"type": "string"}, "queryFinished": {"type": "string"}, - "initFinished": {"type": "string"} + "initFinished": {"type": "string"}, + "queries": {"$ref": "#/definitions/queries"} }, "additionalProperties": {"type": "undefined"}, - "required": ["className", "threads", "queriesFile", "queryError", "queryFinished", "initFinished"] + "required": ["className", "threads", "queryError", "queryFinished", "initFinished", "queries"] } }, { @@ -157,7 +201,6 @@ "properties": { "className": {"type": "string"}, "threads": {"type": "integer"}, - "queriesFile": {"type": "string"}, "timeOut": {"type": "integer"}, "fixedLatency": {"type": "integer"}, "gaussianLatency": {"type": "integer"}, @@ -166,25 +209,27 @@ "queryFinished": {"type": "string"}, "initFinished": {"type": "string"}, "querySuffix": {"type": "string"}, - "queryPrefix": {"type": "string"} + "queryPrefix": {"type": "string"}, + "queries": {"$ref": "#/definitions/queries"} }, "additionalProperties": {"type": "undefined"}, "required": [ "className", "threads", - "queriesFile", "queryError", "queryFinished", "initFinished", "queryPrefix", - "querySuffix"] + "querySuffix", + "queries" + ] } }, { "if": { - "properties": { - "className": { - "oneOf": [{"const": "MultipleCLIInputFileWorker"}, {"const": "org.aksw.iguana.cc.worker.impl.MultipleCLIInputFileWorker"}] + "properties": { + "className": { + "oneOf": [{"const": "MultipleCLIInputFileWorker"}, {"const": "org.aksw.iguana.cc.worker.impl.MultipleCLIInputFileWorker"}] } } }, @@ -192,7 +237,6 @@ "properties": { "className": {"type": "string"}, "threads": {"type": "integer"}, - "queriesFile": {"type": "string"}, "timeOut": {"type": "integer"}, "fixedLatency": {"type": "integer"}, "gaussianLatency": {"type": "integer"}, @@ -200,17 +244,18 @@ "queryFinished": {"type": "string"}, "initFinished": {"type": "string"}, "directory": {"type": "string"}, - "numberOfProcesses": {"type": "integer"} + "numberOfProcesses": {"type": "integer"}, + "queries": {"$ref": "#/definitions/queries"} }, "additionalProperties": {"type": "undefined"}, "required": [ "className", "threads", - "queriesFile", "directory", "queryError", "queryFinished", - "initFinished" + "initFinished", + "queries" ] } }, @@ -224,31 +269,31 @@ }, "then": { "allOf": [ - { - "required": [ - "className", - "threads", - "queriesFile", - "directory", - "queryError", - "queryFinished", - "initFinished" - ] - }, { "properties": { "className": {"type": "string"}, "threads": {"type": "integer"}, - "queriesFile": {"type": "string"}, "timeOut": {"type": "integer"}, "fixedLatency": {"type": "integer"}, "gaussianLatency": {"type": "integer"}, "queryError": {"type": "string"}, "queryFinished": {"type": "string"}, "initFinished": {"type": "string"}, - "directory": {"type": "string"} + "directory": {"type": "string"}, + "queries": {"$ref": "#/definitions/queries"} }, "additionalProperties": {"type": "undefined"} + }, + { + "required": [ + "className", + "threads", + "directory", + "queryError", + "queryFinished", + "initFinished", + "queries" + ] } ] } From e91b181902286b33660f91657a80a1ff02deec8e Mon Sep 17 00:00:00 2001 From: nck-mlcnv <32801560+nck-mlcnv@users.noreply.github.com> Date: Fri, 17 Mar 2023 12:27:34 +0100 Subject: [PATCH 31/47] added endpoint as requirement for pattern key in the schema file --- .../src/main/resources/iguana-schema.json | 17 +++++++++-------- schema/iguana-schema.json | 17 +++++++++-------- 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/iguana.corecontroller/src/main/resources/iguana-schema.json b/iguana.corecontroller/src/main/resources/iguana-schema.json index 4374a3c5b..71b45a045 100644 --- a/iguana.corecontroller/src/main/resources/iguana-schema.json +++ b/iguana.corecontroller/src/main/resources/iguana-schema.json @@ -53,7 +53,8 @@ "endpoint": {"type": "string"}, "outputFolder": {"type": "string"}, "limit": {"type": "integer"} - } + }, + "required": ["endpoint"] }, "lang": {"type": "string"} }, @@ -115,7 +116,7 @@ "parameterName": {"type": "string"}, "queries": {"$ref": "#/definitions/queries"} }, - "additionalProperties": {"type": "undefined"}, + "additionalProperties": {"type": "null"}, "required": ["className", "threads", "queries"] } }, @@ -137,7 +138,7 @@ "timerStrategy": {"type": "string"}, "queries": {"$ref": "#/definitions/queries"} }, - "additionalProperties": {"type": "undefined"}, + "additionalProperties": {"type": "null"}, "required": ["className", "threads", "queries"] } }, @@ -161,7 +162,7 @@ "numberOfProcesses": {"type": "integer"}, "queries": {"$ref": "#/definitions/queries"} }, - "additionalProperties": {"type": "undefined"}, + "additionalProperties": {"type": "null"}, "required": ["className", "threads", "queryError", "queryFinished", "initFinished", "queries"] } }, @@ -185,7 +186,7 @@ "initFinished": {"type": "string"}, "queries": {"$ref": "#/definitions/queries"} }, - "additionalProperties": {"type": "undefined"}, + "additionalProperties": {"type": "null"}, "required": ["className", "threads", "queryError", "queryFinished", "initFinished", "queries"] } }, @@ -212,7 +213,7 @@ "queryPrefix": {"type": "string"}, "queries": {"$ref": "#/definitions/queries"} }, - "additionalProperties": {"type": "undefined"}, + "additionalProperties": {"type": "null"}, "required": [ "className", "threads", @@ -247,7 +248,7 @@ "numberOfProcesses": {"type": "integer"}, "queries": {"$ref": "#/definitions/queries"} }, - "additionalProperties": {"type": "undefined"}, + "additionalProperties": {"type": "null"}, "required": [ "className", "threads", @@ -282,7 +283,7 @@ "directory": {"type": "string"}, "queries": {"$ref": "#/definitions/queries"} }, - "additionalProperties": {"type": "undefined"} + "additionalProperties": {"type": "null"} }, { "required": [ diff --git a/schema/iguana-schema.json b/schema/iguana-schema.json index 4374a3c5b..71b45a045 100644 --- a/schema/iguana-schema.json +++ b/schema/iguana-schema.json @@ -53,7 +53,8 @@ "endpoint": {"type": "string"}, "outputFolder": {"type": "string"}, "limit": {"type": "integer"} - } + }, + "required": ["endpoint"] }, "lang": {"type": "string"} }, @@ -115,7 +116,7 @@ "parameterName": {"type": "string"}, "queries": {"$ref": "#/definitions/queries"} }, - "additionalProperties": {"type": "undefined"}, + "additionalProperties": {"type": "null"}, "required": ["className", "threads", "queries"] } }, @@ -137,7 +138,7 @@ "timerStrategy": {"type": "string"}, "queries": {"$ref": "#/definitions/queries"} }, - "additionalProperties": {"type": "undefined"}, + "additionalProperties": {"type": "null"}, "required": ["className", "threads", "queries"] } }, @@ -161,7 +162,7 @@ "numberOfProcesses": {"type": "integer"}, "queries": {"$ref": "#/definitions/queries"} }, - "additionalProperties": {"type": "undefined"}, + "additionalProperties": {"type": "null"}, "required": ["className", "threads", "queryError", "queryFinished", "initFinished", "queries"] } }, @@ -185,7 +186,7 @@ "initFinished": {"type": "string"}, "queries": {"$ref": "#/definitions/queries"} }, - "additionalProperties": {"type": "undefined"}, + "additionalProperties": {"type": "null"}, "required": ["className", "threads", "queryError", "queryFinished", "initFinished", "queries"] } }, @@ -212,7 +213,7 @@ "queryPrefix": {"type": "string"}, "queries": {"$ref": "#/definitions/queries"} }, - "additionalProperties": {"type": "undefined"}, + "additionalProperties": {"type": "null"}, "required": [ "className", "threads", @@ -247,7 +248,7 @@ "numberOfProcesses": {"type": "integer"}, "queries": {"$ref": "#/definitions/queries"} }, - "additionalProperties": {"type": "undefined"}, + "additionalProperties": {"type": "null"}, "required": [ "className", "threads", @@ -282,7 +283,7 @@ "directory": {"type": "string"}, "queries": {"$ref": "#/definitions/queries"} }, - "additionalProperties": {"type": "undefined"} + "additionalProperties": {"type": "null"} }, { "required": [ From 3d1400a8d3f5b2c1315e059e4f3ba918c0947b6f Mon Sep 17 00:00:00 2001 From: nck-mlcnv <32801560+nck-mlcnv@users.noreply.github.com> Date: Fri, 17 Mar 2023 12:30:58 +0100 Subject: [PATCH 32/47] add missing dependency for PatternHandler --- iguana.corecontroller/pom.xml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/iguana.corecontroller/pom.xml b/iguana.corecontroller/pom.xml index 384c7858d..d96f94abf 100644 --- a/iguana.corecontroller/pom.xml +++ b/iguana.corecontroller/pom.xml @@ -51,6 +51,11 @@ httpclient 4.5.13 + + commons-codec + commons-codec + 1.15 + com.fasterxml.jackson.dataformat jackson-dataformat-yaml From 6a6e9e8aa3723ae7d00fc8b3a0957e02523ba23d Mon Sep 17 00:00:00 2001 From: nck-mlcnv <32801560+nck-mlcnv@users.noreply.github.com> Date: Fri, 17 Mar 2023 12:37:59 +0100 Subject: [PATCH 33/47] add javadocs for PatternHandler --- .../cc/query/pattern/PatternHandler.java | 51 +++++++++++++++---- 1 file changed, 42 insertions(+), 9 deletions(-) diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/pattern/PatternHandler.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/pattern/PatternHandler.java index 9573beed3..5194ddb6b 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/pattern/PatternHandler.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/pattern/PatternHandler.java @@ -49,13 +49,15 @@ public QuerySource generateQuerySource() { File cacheFile = new File(this.outputFolder + File.separator + this.querySource.hashCode()); if (cacheFile.exists()) { - LOGGER.warn("Output folder already exists. Will not generate queries again. To generate them new remove the {{}} folder", cacheFile.getAbsolutePath()); + LOGGER.warn("Output file already exists. Will not generate queries again. To generate them new remove the {{}} file", cacheFile.getAbsolutePath()); } else { LOGGER.info("Generating queries for pattern queries"); File outFolder = new File(this.outputFolder); if (!outFolder.exists()) { - outFolder.mkdirs(); + if(!outFolder.mkdirs()) { + LOGGER.error("Failed to create folder for the generated queries"); + } } try (PrintWriter pw = new PrintWriter(cacheFile)) { @@ -94,6 +96,12 @@ private void init() { } } + /** + * This method generates a list of queries for a given pattern query. + * + * @param query String of the pattern query + * @return List of generated queries as strings + */ protected List generateQueries(String query) { List queries = new LinkedList<>(); @@ -106,18 +114,20 @@ protected List generateQueries(String query) { } catch (Exception ignored) { } - //get vars from queryStr + // Replace the pattern variables with real variables and store them to the Set varNames Set varNames = new HashSet<>(); String command = replaceVars(query, varNames); - //generate parameterized sparql query + // Generate parameterized sparql string to construct final queries ParameterizedSparqlString pss = new ParameterizedSparqlString(); pss.setCommandText(command); + ResultSet exchange = getInstanceVars(pss, varNames); + // exchange vars in PSS if (!exchange.hasNext()) { //no solution - LOGGER.warn("Query has no solution, will use variables instead of var instances: {{}}", query); + LOGGER.warn("Pattern query has no solution, will use variables instead of var instances: {{}}", pss); queries.add(command); } while (exchange.hasNext()) { @@ -134,25 +144,48 @@ protected List generateQueries(String query) { return queries; } + /** + * Replaces the pattern variables of the pattern query with actual variables and returns it. + * The names of the replaced variables will be stored in the set. + * + * @param queryStr String of the pattern query + * @param varNames This set will be extended by the strings of the replaced variable names + * @return The pattern query with the actual variables instead of pattern variables + */ protected String replaceVars(String queryStr, Set varNames) { String command = queryStr; Pattern pattern = Pattern.compile("%%var[0-9]+%%"); Matcher m = pattern.matcher(queryStr); while (m.find()) { - String eVar = m.group(); - String var = eVar.replace("%", ""); - command = command.replace(eVar, "?" + var); + String patternVariable = m.group(); + String var = patternVariable.replace("%", ""); + command = command.replace(patternVariable, "?" + var); varNames.add(var); } return command; } + /** + * Generates valid values for the given variables in the query. + * + * @param pss The query, whose variables should be instantiated + * @param varNames The set of variables in the query that should be instantiated + * @return ResultSet that contains valid values for the given variables of the query + */ protected ResultSet getInstanceVars(ParameterizedSparqlString pss, Set varNames) { QueryExecution exec = QueryExecutionFactory.createServiceRequest(this.endpoint, convertToSelect(pss, varNames)); //return result set return exec.execSelect(); } + /** + * Creates a new query that can find valid values for the variables in the original query. + * The variables, that should be instantiated, are named by the set. + * + * @param pss The query whose variables should be instantiated + * @param varNames The set of variables in the given query that should be instantiated + * @return Query that can evaluate valid values for the given variables of the original query + */ protected Query convertToSelect(ParameterizedSparqlString pss, Set varNames) { Query queryCpy; try { @@ -161,7 +194,7 @@ protected Query convertToSelect(ParameterizedSparqlString pss, Set varNa } queryCpy = pss.asQuery(); } catch (Exception e) { - LOGGER.error("Could not convert query to select (is it update query?): {{}}", pss.toString(), e); + LOGGER.error("The pattern query is not a valid SELECT query (is it perhaps an UPDATE query?): {{}}", pss.toString(), e); return null; } From 1897018f57b0eed8de3b9708d1745e0eb38f7b3d Mon Sep 17 00:00:00 2001 From: nck-mlcnv <32801560+nck-mlcnv@users.noreply.github.com> Date: Sat, 18 Mar 2023 17:17:35 +0100 Subject: [PATCH 34/47] fix the tutorial page in the documentation --- docs/usage/tutorial.md | 224 ++++++++++++++++++++++------------------- 1 file changed, 118 insertions(+), 106 deletions(-) diff --git a/docs/usage/tutorial.md b/docs/usage/tutorial.md index 23d4f3cbe..34ac58b4d 100644 --- a/docs/usage/tutorial.md +++ b/docs/usage/tutorial.md @@ -1,55 +1,56 @@ # Tutorial +In this tutorial, we will set up and execute a benchmark that will run a stresstest on two systems with two different datasets. -In this tutorial we will go through one benchmark using two systems, two datasets and one Stresstest. +We will be using `Iguana v4.0.0` and the following two systems: -We are using the following - -* Iguana v4.0.0 -* Apache Jena Fuseki 3 -* Blazegraph +* Apache Jena Fuseki 4.7 +* Blazegraph 2.1.6 ## Download -First lets create a working directory +First, create a working directory: ```bash mkdir myBenchmark cd myBenchmark ``` -Now let's download all required systems and Iguana. +Now you have to download all required systems and Iguana. -Starting with Iguana +You can download Iguana from the GitHub release page by running these commands in bash: ```bash wget https://github.com/dice-group/IGUANA/releases/download/v4.0.0/iguana-4.0.0.zip unzip iguana-4.0.0.zip ``` -Now we will download Blazegraph +Now we will download Blazegraph: ```bash -mkdir blazegraph && cd blazegraph -wget https://downloads.sourceforge.net/project/bigdata/bigdata/2.1.5/blazegraph.jar?r=https%3A%2F%2Fsourceforge.net%2Fprojects%2Fbigdata%2Ffiles%2Fbigdata%2F2.1.5%2Fblazegraph.jar%2Fdownload%3Fuse_mirror%3Dmaster%26r%3Dhttps%253A%252F%252Fwww.blazegraph.com%252Fdownload%252F%26use_mirror%3Dnetix&ts=1602007009 -cd ../ +mkdir blazegraph +cd blazegraph +wget https://github.com/blazegraph/database/releases/download/BLAZEGRAPH_2_1_6_RC/blazegraph.jar +cd .. ``` -At last, we just need to download Apache Jena Fuseki and Apache Jena +At last, we will download Apache Jena Fuseki and Apache Jena: ```bash mkdir fuseki && cd fuseki -wget https://downloads.apache.org/jena/binaries/apache-jena-3.16.0.zip -unzip apache-jena-3.16.0.zip -wget https://downloads.apache.org/jena/binaries/apache-jena-fuseki-3.16.0.zip -unzip apache-jena-fuseki-3.16.0.zip +wget https://dlcdn.apache.org/jena/binaries/apache-jena-fuseki-4.7.0.tar.gz +tar -xvf apache-jena-fuseki-4.7.0.tar.gz + +wget https://dlcdn.apache.org/jena/binaries/apache-jena-4.7.0.tar.gz +tar -xvf apache-jena-4.7.0.tar.gz +cd .. ``` Finally, we have to download our datasets. -We use two small datasets from scholarly data. +We will be using two small datasets from scholarly data. The ISWC 2010 and the ekaw 2012 rich dataset. -``` +```bash mkdir datasets/ cd datasets wget http://www.scholarlydata.org/dumps/conferences/alignments/iswc-2010-complete-alignments.rdf @@ -57,55 +58,73 @@ wget http://www.scholarlydata.org/dumps/conferences/alignments/ekaw-2012-complet cd .. ``` -That's it. -Lets setup blazegraph and fuseki. +## Systems Setup -## Setting Up Systems - -To simplify the benchmark workflow we will use the pre- and post-script hook, in which we will load the current system -and after the benchmark stop the system. +To simplify the benchmark workflow we will use the pre- and post-task script hook, in which we will load the current system with datasets and stop it after the benchmark. ### Blazegraph +Before we can write our scripts, we will first need to create a properties-file for blazegraph's dataloader. To do this, go to the blazegraph folder with and create a file called `p.properties` with: +```bash +cd blazegraph +touch p.properties +``` + +Then, insert this basic configuration, which should suffice for this tutorial, into the `p.properties` file: +``` +com.bigdata.rdf.store.AbstractTripleStore.statementIdentifiers=true +com.bigdata.journal.AbstractJournal.bufferMode=DiskRW +com.bigdata.journal.AbstractJournal.file=blazegraph.jnl +``` -First let's create the script files +Now we can go ahead and create our script files. First create the files with: ```bash -cd blazegraph touch load-and-start.sh touch stop.sh ``` -The `load-and-start.sh` script will start blazegraph and use curl to POST our dataset. -In our case the datasets are pretty small, hence the loading time is minimal. -Otherwise, it would be wise to load the dataset beforehand, backup the `blazegraph.jnl` file and simply exchanging the -file in the pre script hook. - -For now put this into the script `load-and-start.sh` +We will now write our pre-task script into `load-and-start.sh`. The following script will start +blazegraph and load the given datasets: ```bash -#starting blazegraph with 4 GB ram -cd ../blazegraph && java -Xmx4g -server -jar blazegraph.jar & +#!/bin/bash -#load the dataset file in, which will be set as the first script argument -curl -X POST H 'Content-Type:application/rdf+xml' --data-binary '@$1' http://localhost:9999/blazegraph/sparql -``` +cd ../blazegraph -Now edit `stop.sh` and adding the following: +# load the dataset file, which will be set as the first script argument +java -cp blazegraph.jar com.bigdata.rdf.store.DataLoader -defaultGraph http://example.org p.properties $1 +# start blazegraph with 4 GB ram +java -Xmx4g -server -jar blazegraph.jar & + +# give blazegraph time to boot +sleep 10 ``` + +Now edit `stop.sh` and add the following: + +```bash +#!/bin/bash + +cd ../blazegraph + +# stop the blazegraph server pkill -f blazegraph + +# delete the previous dataset +rm -f ./blazegraph.jnl ``` -Be aware that this kills all blazegraph instances, so make sure that no other process which includes the word blazegraph is running. +Be aware that this kills all blazegraph instances, so make sure that no other process, which includes the word blazegraph, is running. -finally get into the correct working directory again +Finally, change the current working directory again: ```bash cd .. ``` ### Fuseki -Now the same for fuseki: +Now we will do the same for fuseki: ```bash cd fuseki @@ -113,64 +132,50 @@ touch load-and-start.sh touch stop.sh ``` -The `load-and-start.sh` script will load the dataset into a TDB directory and start fuseki using the directory. - -Edit the script `load-and-start.sh` as follows +The `load-and-start.sh` script will start fuseki with the given dataset loaded into the memory. +Edit the script `load-and-start.sh` as follows: ```bash -cd ../fuseki -# load the dataset as a tdb directory -apache-jena-3.16.0/bin/tdbloader2 --loc DB $1 +#!/bin/bash -# start fuseki -apache-jena-fuseki-3.16.0/fuseki-server --loc DB /ds & - -``` - -To assure fairness and provide Fuseki with 4GB as well edit `apache-jena-fuseki-3.16.0/fuseki-server` and go to the last bit exchange the following - -``` -JVM_ARGS=${JVM_ARGS:--Xmx1200M} -``` +cd ../fuseki -to +# start fuseki server service in the background +./apache-jena-fuseki-4.7.0/fuseki-server -q --file $1 /ds & -``` -JVM_ARGS=${JVM_ARGS:--Xmx4G} +# sleep to give fuseki time to boot +sleep 10 ``` -Now edit `stop.sh` and adding the following: +Now edit `stop.sh` and add the following: -``` +```bash +#!/bin/bash pkill -f fuseki ``` Be aware that this kills all Fuseki instances, so make sure that no other process which includes the word fuseki is running. -finally get into the correct working directory again +Finally, change the current working directory again: ```bash cd .. ``` ## Benchmark queries -We need some queries to benchmark. - -For now, we will just use 3 simple queries +Now we need some queries to benchmark. For now, we will just use these 3 simple queries: ``` SELECT * {?s ?p ?o} SELECT * {?s ?p ?o} LIMIT 10 SELECT * {?s ?o} ``` -save this to `queries.txt` - - +Save them to `queries.txt`. ## Creating the Benchmark Configuration -Now let's create the Iguana benchmark configuration. -Create a file called `benchmark-suite.yml` +Now, let's create the Iguana benchmark configuration. +Create a file called `benchmark-suite.yml`: ```bash touch benchmark-suite.yml @@ -179,13 +184,12 @@ touch benchmark-suite.yml Add the following subsections to this file, or simply go to [#Full Configuration](configuration#full-example) and add the whole piece to it. -Be aware that the configuration will be started on directory level below our working directory and thus paths will -use `../` to get the correct path. +Be aware that Iguana will be started from the directory `myBenchmark/iguana/`, thus paths will need to use `../` to get the correct paths. ### Datasets We have two datasets, the ekaw 2012 and the iswc 2010 datasets. -Let's name them as such and set the file path, s.t. the script hooks can use the file paths. +Let's name them as such and set the file path, so that the script hooks can use the files. ```yaml datasets: @@ -197,7 +201,7 @@ datasets: ### Connections -We have two connections, blazegraph and fuseki with their respective endpoint at them as following: +We have two connections, blazegraph and fuseki with their respective endpoint: ```yaml connections: @@ -209,28 +213,34 @@ connections: ### Task script hooks -To assure that the correct triple store will be loaded with the correct dataset add the following pre script hook `../{{ '{{connection}}' }}/load-and-start.sh {{ '{{dataset.file}}' }}` -`{{ '{{connection}}' }}` will be set to the current benchmarked connection name (e.g. `fuseki`) and the `{{ '{{dataset.file}}' }}` will be set to the current dataset file path. +To ensure that the correct triple store will be loaded with the correct dataset, add the following `preScriptHook`: + +```yaml +preScriptHook: "../{{connection}}/load-and-start.sh {{dataset.file}}" +``` -For example the start script of fuseki is located at `fuseki/load-and-start.sh`. +This will execute the appropriate script with the current dataset as the argument, before running a task. +`{{connection}}` will be set to the current benchmarked connection name (e.g. `fuseki`) and the `{{dataset.file}}` will be set to the current dataset file path. -Further on add the `stop.sh` script as the post-script hook, assuring that the store will be stopped after each task +For example, the pre-task script execution for fuseki and the ekaw dataset +will look like this: +```bash +./fuseki/load-and-start.sh ../datasets/ekaw-2012-complete-alignments.rdf +``` -This will look like this: +Further on add the `stop.sh` scripts as the `postScriptHook`, ensuring that the triple store will be stopped after each task: ```yaml -pre-script-hook: "../{{ '{{connection}}' }}/load-and-start.sh {{ '{{dataset.file}}' }}" -post-script-hook: "../{{ '{{connection}}' }}/stop.sh +postScriptHook: "../{{connection}}/stop.sh" ``` ### Task configuration -We want to stresstest our stores using 10 minutes (60.000 ms)for each dataset connection pair. -We are using plain text queries (one per line) and want to have two simulated users querying SPARQL queries. -The queries file is located at our working directory at `queries.txt`. Be aware that we start Iguana one level below, -which makes the correct path `../queries.txt` +We want to stresstest our triple stores for 10 minutes (600,000 ms) for each dataset and each connection. +We are storing the queries in a single file with one query per line, and want to have two simulated users querying SPARQL queries. +The queries are located at our working directory at `queries.txt`. -To achieve these restrictions add the following to your file +The configuration for this setup looks like this: ```yaml tasks: @@ -241,14 +251,15 @@ tasks: - threads: 2 className: "HttpGetWorker" queries: + format: "one-per-line" location: "../queries.txt" ``` ### Result Storage -Let's put the results as an NTriple file and for smootheness of this tutorial let's put it into the file `my-first-iguana-results.nt` +Let's save the results as an NTriple file called `my-first-iguana-results.nt`. -Add the following to do this. +Add the following to do this: ```yaml storages: @@ -272,18 +283,19 @@ connections: - name: "fuseki" endpoint: "http://localhost:3030/ds/sparql" -pre-script-hook: "../{{ '{{connection}}' }}/load-and-start.sh {{ '{{dataset.file}}' }}" -post-script-hook: "../{{ '{{connection}}' }}/stop.sh +preScriptHook: "../{{connection}}/load-and-start.sh {{dataset.file}}" +postScriptHook: "../{{connection}}/stop.sh" tasks: - className: "Stresstest" - configuration: - timeLimit: 600000 - workers: - - threads: 2 - className: "HttpGetWorker" - queries: - location: "../queries.txt" + configuration: + timeLimit: 600000 + workers: + - threads: 2 + className: "HttpGetWorker" + queries: + format: "one-per-line" + location: "../queries.txt" storages: - className: "NTFileStorage" @@ -293,7 +305,7 @@ storages: ## Starting Benchmark -Simply use the previous created `benchmark-suite.yml` and start with +Simply use the previous created `benchmark-suite.yml` and start it with: ```bash cd iguana/ @@ -308,11 +320,11 @@ As previously shown, our results will be shown in `my-first-iguana-results.nt`. Load this into a triple store of your choice and query for the results you want to use. -Just use blazegraph for example: +You can use blazegraph for example: ```bash cd blazegraph -../load-and-start.sh ../my-first-iguana-results.nt +./load-and-start.sh ../iguana/my-first-iguana-results.nt ``` To query the results go to `http://localhost:9999/blazegraph/`. @@ -330,7 +342,7 @@ SELECT ?taskID ?datasetLabel ?connectionLabel ?noq { ?suiteID rdf:type iont:Suite . ?suiteID iprop:experiment ?expID . ?expID iprop:dataset ?dataset . - ?dataset rdfs:label ?datasetLabel + ?dataset rdfs:label ?datasetLabel . ?expID iprop:task ?taskID . ?taskID iprop:connection ?connection. ?connection rdfs:label ?connectionLabel . @@ -339,7 +351,7 @@ SELECT ?taskID ?datasetLabel ?connectionLabel ?noq { ``` -This will provide a list of all task, naming the dataset, the connection and the no. of queries which were succesfully executed +This will provide a list of all tasks, with their respective dataset, connection, and the number of successfully executed queries. We will however not go into detail on how to read the results. -This can be read at [Benchmark Results](../results/) +Further details can be read at [Benchmark Results](./results). From 09ad332748cfc32ab8e6b35f4b91313e987fef6b Mon Sep 17 00:00:00 2001 From: nck-mlcnv <32801560+nck-mlcnv@users.noreply.github.com> Date: Wed, 22 Mar 2023 11:45:18 +0100 Subject: [PATCH 35/47] update README.md --- README.md | 259 +++++++++++++++++++++++------------------------------- 1 file changed, 111 insertions(+), 148 deletions(-) diff --git a/README.md b/README.md index a83d95f92..72172e3e1 100644 --- a/README.md +++ b/README.md @@ -1,148 +1,111 @@ -[![GitLicense](https://gitlicense.com/badge/dice-group/IGUANA)](https://gitlicense.com/license/dice-group/IGUANA) -![Java CI with Maven](https://github.com/dice-group/IGUANA/workflows/Java%20CI%20with%20Maven/badge.svg)[![BCH compliance](https://bettercodehub.com/edge/badge/AKSW/IGUANA?branch=master)](https://bettercodehub.com/) -[![Codacy Badge](https://api.codacy.com/project/badge/Grade/9668460dd04c411fab8bf5ee9c161124)](https://www.codacy.com/app/TortugaAttack/IGUANA?utm_source=github.com&utm_medium=referral&utm_content=AKSW/IGUANA&utm_campaign=Badge_Grade) -[![Project Stats](https://www.openhub.net/p/iguana-benchmark/widgets/project_thin_badge.gif)](https://www.openhub.net/p/iguana-benchmark) - - -# IGUANA - -IGUANA Logo - -## ABOUT - - -Semantic Web is becoming more important and it's data is growing each day. Triple stores are the backbone here, managing these data. -Hence it is very important that the triple store must scale on the data and can handle several users. -Current Benchmark approaches could not provide a realistic scenario on realistic data and could not be adjustet for your needs very easily. -Additionally Question Answering systems and Natural Language Processing systems are becoming more and more popular and thus needs to be stresstested as well. -Further on it was impossible to compare results for different benchmarks. - -Iguana is an an Integerated suite for benchmarking read/write performance of HTTP endpoints and CLI Applications.
which solves all these issues. -It provides an enviroment which ... - - -+ ... is highly configurable -+ ... provides a realistic scneario benchmark -+ ... works on every dataset -+ ... works on SPARQL HTTP endpoints -+ ... works on HTTP Get & Post endpoints -+ ... works on CLI applications -+ and is easily extendable - - -For further Information visit - -[iguana-benchmark.eu](http://iguana-benchmark.eu) - -[Documentation](http://iguana-benchmark.eu/docs/3.3/) - - -# Getting Started - -# Prerequisites - -You need to install Java 11 or greater. -In Ubuntu you can install these using the following commands - -``` -sudo apt-get install java -``` - -# Iguana Modules - -Iguana consists of two modules - -1. **corecontroller**: This will benchmark the systems -2. **resultprocessor**: This will calculate the Metrics and save the raw benchmark results - -## **corecontroller** - -The **corecontroller** will benchmark your system. It should be started on the same machine the is started. - -## **resultprocessor** - -The **resultprocessor** will calculate the metrics. -By default it stores its result in a ntriple file. But you may configure it, to write the results directly to a Triple Store. -On the processing side, it calculates various metrics. - -Per run metrics: -* Query Mixes Per Hour (QMPH) -* Number of Queries Per Hour (NoQPH) -* Number of Queries (NoQ) -* Average Queries Per Second (AvgQPS) - -Per query metrics: -* Queries Per Second (QPS) - * Number of successful and failed queries - * result size - * queries per second - * sum of execution times - -You can change these in the Iguana Benchmark suite config. - -If you use the [basic configuration](https://github.com/dice-group/IGUANA/blob/master/example-suite.yml), it will save all mentioned metrics to a file called `results_{{DATE_RP_STARTED}}.nt` - - -# Setup Iguana - -## Download -Please download the release zip **iguana-x.y.z.zip** from the newest release available [here](https://github.com/dice-group/IGUANA/releases/latest): - -``` -mkdir iguana -wget https://github.com/dice-group/IGUANA/releases/download/v4.0.0/iguana-4.0.0.zip -unzip iguana-4.0.0.zip -``` - - -It contains the following files: - -* iguana.corecontroller-X.Y.Z.jar -* start-iguana.sh -* example-suite.yml - -# Run Your Benchmarks - -## Create a Configuration - -You can use the [basic configuration](https://github.com/dice-group/IGUANA/blob/master/example-suite.yml) we provide and modify it to your needs. -For further information please visit our [configuration](http://iguana-benchmark.eu/docs/3.2/usage/configuration/) and [Stresstest](http://iguana-benchmark.eu/docs/3.0/usage/stresstest/) wiki pages. For a detailed, step-by-step instruction please attend our [tutorial](http://iguana-benchmark.eu/docs/3.2/usage/tutorial/). - - - -## Execute the Benchmark - -Use the start script -``` -./start-iguana.sh example-suite.yml -``` -Now Iguana will execute the example benchmark suite configured in the example-suite.yml file - - -# How to Cite - -```bibtex -@InProceedings{10.1007/978-3-319-68204-4_5, -author="Conrads, Lixi -and Lehmann, Jens -and Saleem, Muhammad -and Morsey, Mohamed -and Ngonga Ngomo, Axel-Cyrille", -editor="d'Amato, Claudia -and Fernandez, Miriam -and Tamma, Valentina -and Lecue, Freddy -and Cudr{\'e}-Mauroux, Philippe -and Sequeda, Juan -and Lange, Christoph -and Heflin, Jeff", -title="Iguana: A Generic Framework for Benchmarking the Read-Write Performance of Triple Stores", -booktitle="The Semantic Web -- ISWC 2017", -year="2017", -publisher="Springer International Publishing", -address="Cham", -pages="48--65", -abstract="The performance of triples stores is crucial for applications driven by RDF. Several benchmarks have been proposed that assess the performance of triple stores. However, no integrated benchmark-independent execution framework for these benchmarks has yet been provided. We propose a novel SPARQL benchmark execution framework called Iguana. Our framework complements benchmarks by providing an execution environment which can measure the performance of triple stores during data loading, data updates as well as under different loads and parallel requests. Moreover, it allows a uniform comparison of results on different benchmarks. We execute the FEASIBLE and DBPSB benchmarks using the Iguana framework and measure the performance of popular triple stores under updates and parallel user requests. We compare our results (See https://doi.org/10.6084/m9.figshare.c.3767501.v1) with state-of-the-art benchmarking results and show that our benchmark execution framework can unveil new insights pertaining to the performance of triple stores.", -isbn="978-3-319-68204-4" -} -``` +# IGUANA + +![Java CI with Maven](https://github.com/dice-group/IGUANA/workflows/Java%20CI%20with%20Maven/badge.svg) + +IGUANA Logo + +Iguana is an integrated suite for benchmarking the read/write performance of HTTP endpoints and CLI Applications. + +It provides an environment which ... + +* is highly configurable +* provides a realistic scenario benchmark +* works on every dataset +* works on SPARQL HTTP endpoints +* works on HTTP Get & Post endpoints +* works on CLI applications +* and is easily extendable + +For further information visit: +- [iguana-benchmark.eu](http://iguana-benchmark.eu) +- [Documentation](http://iguana-benchmark.eu/docs/3.3/) + +## Iguana Modules + +Iguana consists of two modules +- **corecontroller** - this will benchmark the systems +- **resultprocessor** - this will calculate the metrics and save the raw benchmark results + +### Available metrics + +Per run metrics: +* Query Mixes Per Hour (QMPH) +* Number of Queries Per Hour (NoQPH) +* Number of Queries (NoQ) +* Average Queries Per Second (AvgQPS) + +Per query metrics: +* Queries Per Second (QPS) +* number of successful and failed queries +* result size +* queries per second +* sum of execution times + +## Setup Iguana + +### Prerequisites + +In order to run Iguana, you need to have `Java 11`, or greater, installed on your system. + +### Download +Download the newest release of Iguana [here](https://github.com/dice-group/IGUANA/releases/latest), or run on a unix shell: + +```sh +wget https://github.com/dice-group/IGUANA/releases/download/v4.0.0/iguana-4.0.0.zip +unzip iguana-4.0.0.zip +``` + +The zip file contains the following files: + +* `iguana-X.Y.Z.jar` +* `start-iguana.sh` +* `example-suite.yml` + +### Create a Configuration + +You can use the provided example configuration and modify it to your needs. +For further information please visit our [configuration](http://iguana-benchmark.eu/docs/3.2/usage/configuration/) and [Stresstest](http://iguana-benchmark.eu/docs/3.0/usage/stresstest/) wiki pages. + +For a detailed, step-by-step instruction through a benchmarking example, please visit our [tutorial](http://iguana-benchmark.eu/docs/3.2/usage/tutorial/). + +### Execute the Benchmark + +Start Iguana with a benchmark suite (e.g. the example-suite.yml) either by using the start script: + +```sh +./start-iguana.sh example-suite.yml +``` + +or by directly executing the jar-file: + +```sh +java -jar iguana-x-y-z.jar example-suite.yml +``` + +# How to Cite + +```bibtex +@InProceedings{10.1007/978-3-319-68204-4_5, +author="Conrads, Lixi +and Lehmann, Jens +and Saleem, Muhammad +and Morsey, Mohamed +and Ngonga Ngomo, Axel-Cyrille", +editor="d'Amato, Claudia +and Fernandez, Miriam +and Tamma, Valentina +and Lecue, Freddy +and Cudr{\'e}-Mauroux, Philippe +and Sequeda, Juan +and Lange, Christoph +and Heflin, Jeff", +title="Iguana: A Generic Framework for Benchmarking the Read-Write Performance of Triple Stores", +booktitle="The Semantic Web -- ISWC 2017", +year="2017", +publisher="Springer International Publishing", +address="Cham", +pages="48--65", +abstract="The performance of triples stores is crucial for applications driven by RDF. Several benchmarks have been proposed that assess the performance of triple stores. However, no integrated benchmark-independent execution framework for these benchmarks has yet been provided. We propose a novel SPARQL benchmark execution framework called Iguana. Our framework complements benchmarks by providing an execution environment which can measure the performance of triple stores during data loading, data updates as well as under different loads and parallel requests. Moreover, it allows a uniform comparison of results on different benchmarks. We execute the FEASIBLE and DBPSB benchmarks using the Iguana framework and measure the performance of popular triple stores under updates and parallel user requests. We compare our results (See https://doi.org/10.6084/m9.figshare.c.3767501.v1) with state-of-the-art benchmarking results and show that our benchmark execution framework can unveil new insights pertaining to the performance of triple stores.", +isbn="978-3-319-68204-4" +} +``` \ No newline at end of file From 7e79bb4190476a371ef9deebabfa514391ce9ca7 Mon Sep 17 00:00:00 2001 From: nck-mlcnv <32801560+nck-mlcnv@users.noreply.github.com> Date: Thu, 23 Mar 2023 13:41:57 +0100 Subject: [PATCH 36/47] update docs --- README.md | 5 +- docs/about.md | 28 ++-- docs/architecture.md | 54 ++++---- docs/develop/extend-lang.md | 122 +++++++----------- docs/develop/extend-metrics.md | 124 +++++++++--------- docs/develop/extend-queryhandling.md | 122 +++++++++++++----- docs/develop/extend-result-storages.md | 36 +++--- docs/develop/extend-task.md | 162 ++++++++++------------- docs/develop/extend-workers.md | 81 ++++++------ docs/develop/maven.md | 13 +- docs/develop/overview.md | 22 ++-- docs/download.md | 14 +- docs/index.md | 14 +- docs/quick-config.md | 20 ++- docs/run-iguana.md | 18 +-- docs/shorthand-mapping.md | 54 ++++---- docs/usage/configuration.md | 171 +++++++++++++------------ docs/usage/getting-started.md | 50 +++++--- docs/usage/languages.md | 10 +- docs/usage/metrics.md | 26 ++-- docs/usage/queries.md | 38 +++--- docs/usage/results.md | 32 ++--- docs/usage/stresstest.md | 114 ++++++++--------- docs/usage/tutorial.md | 10 +- docs/usage/workers.md | 86 ++++++------- docs/usage/workflow.md | 4 +- 26 files changed, 722 insertions(+), 708 deletions(-) diff --git a/README.md b/README.md index 72172e3e1..52a9ab3ab 100644 --- a/README.md +++ b/README.md @@ -2,8 +2,9 @@ ![Java CI with Maven](https://github.com/dice-group/IGUANA/workflows/Java%20CI%20with%20Maven/badge.svg) -IGUANA Logo - +

+ IGUANA Logo +

Iguana is an integrated suite for benchmarking the read/write performance of HTTP endpoints and CLI Applications. It provides an environment which ... diff --git a/docs/about.md b/docs/about.md index 37e2a0ccc..3f89360ce 100644 --- a/docs/about.md +++ b/docs/about.md @@ -1,11 +1,14 @@ # Iguana -Iguana is an an Integerated suite for benchmarking read/write performance of HTTP endpoints and CLI Applications. -Semantic Web is becoming more important and it's data is growing each day. Triple stores are the backbone here, managing these data. Hence it is very important that the triple store must scale on the data and can handle several users. Current Benchmark approaches could not provide a realistic scenario on realistic data and could not be adjustet for your needs very easily. Additionally Question Answering systems and Natural Language Processing systems are becoming more and more popular and thus needs to be stresstested as well. Further on it was impossible to compare results for different benchmarks. +Iguana is an integrated suite for benchmarking the read/write performance of HTTP endpoints and CLI Applications. -Iguana tries to solve all these issues. It provides an enviroment which ... +Semantic Web is becoming more important and its data is growing each day. Triple stores are the backbone here, managing these data. Hence, it is very important that the triple store must scale on the data and can handle several users. Current Benchmark approaches could not provide a realistic scenario on realistic data and could not be adjusted for your needs very easily. + +Additionally, Question Answering systems and Natural Language Processing systems are becoming more and more popular and thus need to be stresstested as well. Further on it was impossible to compare results for different benchmarks. + +Iguana tries to solve all these issues. It provides an environment which ... * is highly configurable -* provides a realistic scneario benchmark +* provides a realistic scenario benchmark * works on every dataset * works on SPARQL HTTP endpoints * works on HTTP Get & Post endpoints @@ -14,22 +17,21 @@ Iguana tries to solve all these issues. It provides an enviroment which ... ## What is Iguana -Iguana is a HTTP and CLI read/write performance benchmark framework suite. -It can stresstest HTTP get and post endpoints as well as CLI applications using a bunch of simulated users which will bombard the endpoint using queries. -Queries can be anything. SPARQL, SQL, Text and anything else you can fit in one line. +Iguana is an HTTP and CLI read/write performance benchmark framework suite. +It can stresstest HTTP GET and POST endpoints as well as CLI applications using a bunch of simulated users which will flood the endpoint using queries. +Queries can be anything. SPARQL, SQL, Text, etc. ## What can be benchmarked -Iguana is capable of benchmarking and stresstesting the following applications +Iguana is capable of benchmarking and stresstesting the following applications: -* HTTP GET and POST endpoint (e.g. Triple Stores, REST Services, Question Answering endpoints) +* HTTP GET and POST endpoints (e.g. Triple Stores, REST Services, Question Answering endpoints) * CLI Applications which either * exit after every query - * or awaiting input after each query + * await for input after each query ## What Benchmarks are possible -Every simulated User (named Worker in the following) gets a set of queries. -These queries have to be saved in one file, whereas each query is one line. -Hence everything you can fit in one line (e.g a SPARQL query, a text question, an RDF document) can be used as a query and a set of these queries represent the benchmark. +Every simulated user (named worker in the following) gets a set of queries. +These queries (e.g. SPARQL queries, text questions, RDF documents) can be saved in a single file or in a folder with multiple files. A set of these queries represent the benchmark. Iguana will then let every Worker execute these queries against the endpoint. diff --git a/docs/architecture.md b/docs/architecture.md index 01f87b401..2ecb831a7 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -1,31 +1,30 @@ # Architecture -Iguanas architecture is build as generic as possible to ensure that your benchmark can be executed while you only have -to create a configuration file which fits your needs. -So ideally you do not need to code anything and can use Iguana out of the box. +Iguanas architecture is built as generic as possible to ensure that you only have to create a configuration file to execute your benchmark. +So ideally, you do not need to code anything and can use Iguana out of the box. Iguana will parse your Configuration (YAML or JSON format) and will read which Endpoints/Applications you want to benchmark. -What datasets if you have any and what your benchmark should accomplish. -Do you just want to check how good your database/triple store performs against the state of the art? -Does your new version out performs the old version? -Do you want to check read and write performance? +What datasets do you want to use, if you have any, and what should your benchmark accomplish? +Do you just want to check how well your database/triple store performs against the state of the art? +Does your new version outperform the old version? +Do you want to check read and write performances? ... -Whatever you want to do you just need to provide Iguana your tested applications, what to benchmark and which queries to use. +Whatever you want to do you just need to provide Iguana with your applications, what to benchmark, and which queries to use. -Iguana relys mainly on HTTP libraries, the JENA framework and java 11. +Iguana relies mainly on HTTP libraries, the JENA framework, and Java 11. ## Overview -Iguana will read the configuration, parse it and executes for each specified datasets, each specified connection with the benchmark tasks you specified. -After the executions the results will be written as RDF. Either to a NTriple file or directly to a triple store. -The results can be queried itself using SPARQL. +Iguana will read the configuration and parse it. Then it executes, for each specified dataset, each specified connection, the benchmark tasks you specified. +After the executions, the results will be written as RDF in a file or directly to a triple store. +The results themselves can be queried using SPARQL. -Iguana currently consists of on implemented Task, the Stresstest. -However, this task is very configurable and most definetly will met your needs if you want performance measurement. -It starts a user defined amount of Workers, which will try to simulate real users/applications querying your Endpoint/Application. +Iguana currently consists of one implemented Task, the Stresstest. +However, this task is very configurable and will most likely meet your demands if you need performance measurements. +It starts a predefined amount of workers, which will try to simulate real users/applications querying your endpoint/application. ## Components @@ -34,17 +33,18 @@ Iguana consists of two components, the core controller and the result processor. ### **core controller** -The core which implements the Tasks and Workers to use. How HTTP responses should be handled. -How to analyze the benchmark queries to give a little bit more extra information in the results. - +The core controller implements the tasks and workers you want to use. It also specifies how HTTP responses should be handled and how the benchmark queries should be analyzed to gain additional information for the results. ### **result processor** -The result processor consist of the metrics to apply to the query execution results and how to save the results. -Most of the SOtA metrics are implemented in Iguana. If one's missing it is pretty easy to add a metric though. +The result processor consists of the metrics, that should be applied to the query execution results, and specifies how to save these results. +Most of the SOtA metrics are implemented in Iguana. If a metric should be missing, it can be easily added to Iguana. + +By default, the result processor stores its results in a file. But you may configure it, to write the results directly to a triple store. + +On the processing side, the result processor calculates various metrics. -By default it stores its result in a ntriple file. But you may configure it, to write the results directly to a Triple Store. -On the processing side, it calculates various metrics. +#### Available metrics Per run metrics: * Query Mixes Per Hour (QMPH) @@ -54,12 +54,12 @@ Per run metrics: Per query metrics: * Queries Per Second (QPS) - * Number of successful and failed queries - * result size - * queries per second - * sum of execution times +* Number of successful and failed queries +* result size +* queries per second +* sum of execution times -You can change these in the Iguana Benchmark suite config. +You can change these in the Iguana benchmark suite configuration. If you use the [basic configuration](https://github.com/dice-group/IGUANA/blob/master/example-suite.yml), it will save all mentioned metrics to a file called `results_{DD}-{MM}-{YYYY}_{HH}-{mm}.nt` diff --git a/docs/develop/extend-lang.md b/docs/develop/extend-lang.md index 4285fdd8a..b9acad12c 100644 --- a/docs/develop/extend-lang.md +++ b/docs/develop/extend-lang.md @@ -1,42 +1,24 @@ # Extend Languages -If you want to add query specific statistics and/or using the correct result size for an HTTP Worker (Post or Get) you can do so. -(This may be interesting if you're not using SPARQL) +If you want to add query specific statistics, that uses the correct result size for an HTTP POST/GET worker, you can implement a `LanguageProcessor`. +(This may be interesting if you're not using SPARQL queries) -Let's start by implementing the `LanguageProcessor` +Let's start by implementing the `LanguageProcessor` interface: ```java @Shorthand("lang.MyLanguage") -public class MyLanguageProcessor implements LanguageProcessor { - - @Override - public String getQueryPrefix() { - } - - - @Override - public Model generateTripleStats(List queries, String resourcePrefix, String taskID) { - } - - @Override - public Long getResultSize(CloseableHttpResponse response) throws ParserConfigurationException, SAXException, ParseException, IOException { - } - - @Override - Long getResultSize(Header contentTypeHeader, BigByteArrayOutputStream content) throws ParserConfigurationException, SAXException, ParseException, IOException{ - } - - @Override - long readResponse(InputStream inputStream, BigByteArrayOutputStream responseBody) throws IOException{ - } - - +public class MyLanguageProcessor implements LanguageProcessor { + // ... } ``` +This class also utilizes the `Shorthand` annotation, for a shorter name in the configuration file. + +In the following, you can find more detailed explanations for the interface methods. + ## Query prefix -Set a query prefix which will be used in the result set, f.e. "sql" +Sets a query prefix, which will be used in the result set, for example "sql": ```java @Override @@ -47,64 +29,58 @@ Set a query prefix which will be used in the result set, f.e. "sql" ## Generate Query Statistics -Generating query specific statistics (which will be added in the result file) +Generates query specific statistics (which will be added in the result file). -You will get the queries (containg of an ID and the query itself) a resourcePrefix you may use to create the URIs and the current taskID. +This method receives a list of all queries as QueryWrappers (the wrapper contains an ID and the query itself), a resourcePrefix, which you may use to create the URIs, and the current taskID. -A basic pretty standard exmaple is +This is what an example may look like: ```java - @Override - public Model generateTripleStats(List queries, String resourcePrefix, String taskID) { - Model model = ModelFactory.createDefaultModel(); - for(QueryWrapper wrappedQuery : queries) { - Resource subject = ResourceFactory.createResource(COMMON.RES_BASE_URI + resourcePrefix + "/" + wrappedQuery.getId()); - model.add(subject, RDF.type, Vocab.queryClass); - model.add(subject, Vocab.rdfsID, wrappedQuery.getId().replace(queryPrefix, "").replace("sql", "")); - model.add(subject, RDFS.label, wrappedQuery.getQuery().toString()); - - //ADD YOUR TRIPLES HERE which contains query specific statistics - } - return model; - +@Override +public Model generateTripleStats(List queries, String resourcePrefix, String taskID) { + Model model = ModelFactory.createDefaultModel(); + for(QueryWrapper wrappedQuery : queries) { + Resource subject = ResourceFactory.createResource(COMMON.RES_BASE_URI + resourcePrefix + "/" + wrappedQuery.getId()); + model.add(subject, RDF.type, Vocab.queryClass); + model.add(subject, Vocab.rdfsID, wrappedQuery.getId().replace(queryPrefix, "").replace("sql", "")); + model.add(subject, RDFS.label, wrappedQuery.getQuery().toString()); + + //ADD YOUR TRIPLES HERE which contains query specific statistics } + return model; +} ``` ## Get the result size -To generate the correct result size in the result file do the following +To generate the correct result size in the result file do the following: ```java - @Override - public Long getResultSize(CloseableHttpResponse response) throws ParserConfigurationException, SAXException, ParseException, IOException { - - - InputStream inStream = response.getEntity().getContent(); - Long size = -1L; - //READ INSTREAM ACCORDINGLY - - - return size; - } - +@Override +public Long getResultSize(CloseableHttpResponse response) throws ParserConfigurationException, SAXException, ParseException, IOException { + InputStream inStream = response.getEntity().getContent(); + Long size = -1L; + + // read the response with the inputstream accordingly + + return size; +} - @Override - public Long getResultSize(Header contentTypeHeader, BigByteArrayOutputStream content) throws ParserConfigurationException, SAXException, ParseException, IOException { - //Read content from Byte Array instead of InputStream - InputStream is = new BigByteArrayInputStream(content); - Long size=-1L; - ... +@Override +public Long getResultSize(Header contentTypeHeader, BigByteArrayOutputStream content) throws ParserConfigurationException, SAXException, ParseException, IOException { + InputStream is = new BigByteArrayInputStream(content); + Long size = -1L; + + // read content from Byte Array instead of InputStream - return size; - } + return size; +} - @Override - public long readResponse(InputStream inputStream, BigByteArrayOutputStream responseBody) throws IOException { - //simply moves content from inputStream to the byte array responseBody and returns the size; - //will be used for parsing the anwser in another thread. - return Streams.inputStream2ByteArrayOutputStream(inputStream, responseBody); - } - - +@Override +public long readResponse(InputStream inputStream, BigByteArrayOutputStream responseBody) throws IOException { + //simply moves content from inputStream to the byte array responseBody and returns the size; + //will be used for parsing the anwser in another thread. + return Streams.inputStream2ByteArrayOutputStream(inputStream, responseBody); +} ``` diff --git a/docs/develop/extend-metrics.md b/docs/develop/extend-metrics.md index cd5667d36..5c4b98331 100644 --- a/docs/develop/extend-metrics.md +++ b/docs/develop/extend-metrics.md @@ -1,17 +1,16 @@ # Extend Metrics -Developed a new metric or simply want to use one that isn't implemented? - -Start by extending the `AbstractMetric` +To implement a new metric, create a new class that extends the abstract class `AbstractMetric`: ```java -package org.benchmark.metric +package org.benchmark.metric; @Shorthand("MyMetric") public class MyMetric extends AbstractMetric{ @Override public void receiveData(Properties p) { + // ... } @Override @@ -22,7 +21,7 @@ public class MyMetric extends AbstractMetric{ } protected void callbackClose() { - //ADD YOUR CLOSING HERE + // your close method } } ``` @@ -31,77 +30,78 @@ public class MyMetric extends AbstractMetric{ This method will receive all the results during the benchmark. -You'll receive a few values regarding that one query execution, the time it took, if it succeeded, if not if it was a timeout, a wrong HTTP Code or unkown. -Further on the result size of the query. +You'll receive a few values regarding each query execution. Those values include the amount of time the execution took, if it succeeded, and if not, the reason why it failed, which can be either a timeout, a wrong HTTP Code or an unknown error. +Further on you also receive the result size of the query. -If your metric is a single value metric you can use the `processData` method, which will automatically add each value together. -However if your metric is query specific you can use the `addDataToContainter` method. (Look at the [QPSMetric](https://github.com/dice-group/IGUANA/blob/master/iguana.resultprocessor/src/main/java/org/aksw/iguana/rp/metrics/impl/QPSMetric.java). +If your metric is a single value metric, you can use the `processData` method, which will automatically add each value together. +However, if your metric is query specific, you can use the `addDataToContainter` method. (Look at the [QPSMetric](https://github.com/dice-group/IGUANA/blob/master/iguana.resultprocessor/src/main/java/org/aksw/iguana/rp/metrics/impl/QPSMetric.java)) -Be aware that both mehtods will save the results for each worker used. This allows to calcualte the overall metric as well the metric for each worker itself. +Be aware that both methods will save the results for each used worker. This allows the calculation of the overall metric, as well as the metric for each worker itself. -We will go with the single-value metric for now. +We will stick to the single-value metric for now. -An example on how to retrieve every possible value and saving the time and success. +The following shows an example, that retrieves every possible value and saves the time and success: ```java - @Override - public void receiveData(Properties p) { - - double time = Double.parseDouble(p.get(COMMON.RECEIVE_DATA_TIME).toString()); - long tmpSuccess = Long.parseLong(p.get(COMMON.RECEIVE_DATA_SUCCESS).toString()); - long success = tmpSuccess>0?1:0; - long failure = success==1?0:1; - long timeout = tmpSuccess==COMMON.QUERY_SOCKET_TIMEOUT?1:0; - long unknown = tmpSuccess==COMMON.QUERY_UNKNOWN_EXCEPTION?1:0; - long wrongCode = tmpSuccess==COMMON.QUERY_HTTP_FAILURE?1:0; - if(p.containsKey(COMMON.RECEIVE_DATA_SIZE)) { - size = Long.parseLong(p.get(COMMON.RECEIVE_DATA_SIZE).toString()); - } - - Properties results = new Properties(); - results.put(TOTAL_TIME, time); - results.put(TOTAL_SUCCESS, success); - - Properties extra = getExtraMeta(p); - processData(extra, results); - } +@Override +public void receiveData(Properties p) { + + double time = Double.parseDouble(p.get(COMMON.RECEIVE_DATA_TIME).toString()); + long tmpSuccess = Long.parseLong(p.get(COMMON.RECEIVE_DATA_SUCCESS).toString()); + long success = (tmpSuccess > 0) ? 1 : 0; + long failure = (success == 1) ? 0 : 1; + long timeout = (tmpSuccess == COMMON.QUERY_SOCKET_TIMEOUT) ? 1 : 0; + long unknown = (tmpSuccess == COMMON.QUERY_UNKNOWN_EXCEPTION) ? 1 : 0; + long wrongCode = (tmpSuccess == COMMON.QUERY_HTTP_FAILURE) ? 1 : 0; + + if(p.containsKey(COMMON.RECEIVE_DATA_SIZE)) { + size = Long.parseLong(p.get(COMMON.RECEIVE_DATA_SIZE).toString()); + } + + Properties results = new Properties(); + results.put(TOTAL_TIME, time); + results.put(TOTAL_SUCCESS, success); + + Properties extra = getExtraMeta(p); + processData(extra, results); +} ``` - ## Close -In this method you should finally calculate your metric and send the results. +In this method you should calculate your metric and send the results. +An example: ```java - protected void callbackClose() { - //create model to contain results - Model m = ModelFactory.createDefaultModel(); - - Property property = getMetricProperty(); - Double sum = 0.0; - - // Go over each worker and add metric results to model. - for(Properties key : dataContainer.keySet()){ - Double totalTime = (Double) dataContainer.get(key).get(TOTAL_TIME); - Integer success = (Integer) dataContainer.get(key).get(TOTAL_SUCCESS); - Double noOfQueriesPerHour = hourInMS*success*1.0/totalTime; - sum+=noOfQueriesPerHour; - Resource subject = getSubject(key); - m.add(getConnectingStatement(subject)); - m.add(subject, property, ResourceFactory.createTypedLiteral(noOfQueriesPerHour)); - } - - // Add overall metric to model - m.add(getTaskResource(), property, ResourceFactory.createTypedLiteral(sum)); - - //Send data to storage - sendData(m); - } - - +protected void callbackClose() { + // create a model that contains the results + Model m = ModelFactory.createDefaultModel(); + + Property property = getMetricProperty(); + Double sum = 0.0; + + // Go over each worker and add metric results to model + for(Properties key : dataContainer.keySet()){ + Double totalTime = (Double) dataContainer.get(key).get(TOTAL_TIME); + Integer success = (Integer) dataContainer.get(key).get(TOTAL_SUCCESS); + + Double noOfQueriesPerHour = hourInMS * success * 1.0 / totalTime; + sum += noOfQueriesPerHour; + Resource subject = getSubject(key); + + m.add(getConnectingStatement(subject)); + m.add(subject, property, ResourceFactory.createTypedLiteral(noOfQueriesPerHour)); + } + + // Add overall metric to model + m.add(getTaskResource(), property, ResourceFactory.createTypedLiteral(sum)); + + // Send data to storage + sendData(m); +} ``` ## Constructor -The constructor parameters will be provided the same way the Task get's the parameters, thus simply look at [Extend Task](../extend-task). +The constructor parameters are provided the same way as for the tasks. Thus, simply look at the [Extend Task](../extend-task) page. diff --git a/docs/develop/extend-queryhandling.md b/docs/develop/extend-queryhandling.md index 6b1702bc1..3805961b6 100644 --- a/docs/develop/extend-queryhandling.md +++ b/docs/develop/extend-queryhandling.md @@ -1,57 +1,109 @@ # Extend Query Handling -If you want to use another query generating method as the implemented ones you can do so. +Currently, there is no way of extending the query handling without modifying the QueryHandler class. -Start by extend the `AbstractWorkerQueryHandler`. It will split up the generation for UPDATE queries and Request queries. +You can change the way queries are handled by extending the following abstract classes: -```java -package org.benchmark.query +| Class | Function | +|-------------------------|------------------------------------------------------------------| +| `AbstractQuerySelector` | Responsible for selecting the next query a worker should execute | +| `AbstractQuerySource` | Responsible for loading queries | +In the following sections, each extension of a class will be described briefly with the necessary changes to the `QueryHandler` class. For further details, read the corresponding javadocs. -public class MyQueryHandler extends AbstractWorkerQueryHandler{ +## QuerySelector - protected abstract QuerySet[] generateQueries(String queryFileName) { - - } +If you want a different execution order for your queries, you can create a class that extends the class `AbstractQuerySelector` and implements the method `getNextIndex`: - protected abstract QuerySet[] generateUPDATE(String updatePath) { - +```java +public class MyQuerySelector extends AbstractQuerySelector { + public MyQuerySelector(int size) { + super(size); + } + + @Override + public int getNextIndex(){ + // code for selecting the next query a worker should execute } - } - ``` -for simplicity we will only show the `generateQueries` as it is pretty much the same. -However be aware that the `generateUPDATE` will use a directory or file instead of just a query file. - -## Generate Queries +Once you've created your QuerySelector class, you need to decide a value for the key `order` (in this example `"myOrder"`) for the configuration file and update the `initQuerySelector` method inside the `QueryHandler` class: -The class will get a query file containing all the queries. -How you read them and what to do with them is up to you. -You just need to return an array of `QuerySet`s +```java +private void initQuerySelector() { + // ... + + if (orderObj instanceof String) { + String order = (String) orderObj; + if (order.equals("linear")) { + this.querySelector = new LinearQuerySelector(this.querySet.size()); + return; + } + if (order.equals("random")) { + this.querySelector = new RandomQuerySelector(this.querySet.size(), this.workerID); + return; + } + + // add this + if (order.equals("myOrder")) { + this.querySelector = new MyQuerySelector(this.querySet.size(), this.workerID); + return; + } + + LOGGER.error("Unknown order: " + order); + } -A query set is simply a container which contains the name/id of the query as well as the query or several queries (f.e. if they are of the same structure but different values). -For simplicity we assume that we deal with only one query per query set. + // ... +} +``` -Parse your file and for each query create a QuerySet +## QuerySource +If you want to use different source for your queries, you can create a class that extends the class `AbstractQuerySourcer` and implements the following methods: ```java - protected QuerySet[] generateQueries(String queryFileName) { - File queryFile = new File(queryFileName); - List ret = new LinkedList(); - - int id=0; - //TODO parse your queries - ... - - ret.add(new InMemQuerySet(idPrefix+id++, queryString)); - ... +public class MyQuerySource extends AbstractQuerySource { + public MyQuerySource(String filepath) { + // your constructor + // filepath is the value, specified in the "location"-key inside the configuration file + } + + @Override + public int size() { + // returns the amount of queries in the source + } + @Override + public String getQuery(int index) throws IOException { + // retrieves a single query with the specific index + } - return ret.toArray(new QuerySet[]{}); + @Override + public List getAllQueries() throws IOException { + // retrieves every query from the source } +} +``` +Once you have created your QuerySelector class, you need to decide a value for the key `format` (in this example `"myFormat"`) for the configuration file and update the `createQuerySource` method inside the `QueryHandler` class: + +```Java +private QuerySource createQuerySource() { + // ... + else { + switch ((String) formatObj) { + case "one-per-line": + return new FileLineQuerySource(this.location); + case "separator": + return new FileSeparatorQuerySource(this.location); + case "folder": + return new FolderQuerySource(this.location); + + // add this + case "myFormat": + return new MyQuerySource(this.location); + } + } + // ... +} ``` - -This function will parse your query accodringly and add an In Memory QuerySet (another option is a File Based Query Set, where each QuerySet will be stored in a file and IO happens during the benchmark itself. diff --git a/docs/develop/extend-result-storages.md b/docs/develop/extend-result-storages.md index 95ab89779..1632c1785 100644 --- a/docs/develop/extend-result-storages.md +++ b/docs/develop/extend-result-storages.md @@ -1,51 +1,47 @@ -#Extend Result Storages +# Extend Result Storages -If you want to use a different storage than RDF you can extend the storages +If you want to use a different storage other than RDF, you can implement a different storage solution. -However it is highly optimized for RDF so we suggest to work on top of the `TripleBasedStorage` +The current implementation of Iguana is highly optimized for RDF, thus we recommend you to work on top of the `TripleBasedStorage` class: ```java -package org.benchmark.storage +package org.benchmark.storage; @Shorthand("MyStorage") public class MyStorage extends TripleBasedStorage { @Override public void commit() { - + } - @Override public String toString(){ return this.getClass().getSimpleName(); } - } - ``` ## Commit -This should take all the current results, store them and remove them from memory. +This method should take all the current results, store them, and remove them from the memory. You can access the results at the Jena Model `this.metricResults`. For example: ```java - - @Override - public void commit() { - try (OutputStream os = new FileOutputStream(file.toString(), true)) { - RDFDataMgr.write(os, metricResults, RDFFormat.NTRIPLES); - metricResults.removeAll(); - } catch (IOException e) { - LOGGER.error("Could not commit to NTFileStorage.", e); - } - } +@Override +public void commit() { + try (OutputStream os = new FileOutputStream(file.toString(), true)) { + RDFDataMgr.write(os, metricResults, RDFFormat.NTRIPLES); + metricResults.removeAll(); + } catch (IOException e) { + LOGGER.error("Could not commit to NTFileStorage.", e); + } +} ``` ## Constructor -The constructor parameters will be provided the same way the Task get's the parameters, thus simply look at [Extend Task](../extend-task). +The constructor parameters are provided the same way as for the tasks. Thus, simply look at the [Extend Task](../extend-task) page. \ No newline at end of file diff --git a/docs/develop/extend-task.md b/docs/develop/extend-task.md index 6c49a8109..f8ed709f3 100644 --- a/docs/develop/extend-task.md +++ b/docs/develop/extend-task.md @@ -1,10 +1,10 @@ # Extend Tasks -You can extend Iguana with your benchmark task, if the Stresstest doesn't fit your needs. -F.e. you may want to check systems if they answer correctly rather than stresstest them. +You can extend Iguana with your own benchmark task if the Stresstest doesn't suffice your needs. +For example, you may only want to check if a system answers correctly to a query, rather than stresstesting them. -You will need to create your own task either in the Iguana code itself or by using Iguana as a library. -Either way start by extending the AbstractTask. +You will need to create your task either in the Iguana code itself or by using Iguana as a library. +Either way, start by extending the `AbstractTask`: ```java package org.benchmark; @@ -13,10 +13,9 @@ package org.benchmark; public class MyBenchmarkTask extends AbstractTask { } - ``` -You will need to override some functions. For now include them and go through them step by step +You will need to override the following functions as in the example: ```java package org.benchmark; @@ -24,149 +23,135 @@ package org.benchmark; @Shorthand("MyBenchmarkTask") public class MyBenchmarkTask extends AbstractTask { - //Your constructor(s) + // your constructor(s) public MyBenchmarkTask(Integer timeLimit, List workers, Map config) throws FileNotFoundException { } - - //Meta Data (which will be added in the resultsfile) + // metadata (which will be added in the results file) @Override public void addMetaData() { - super.addMetaData(); + super.addMetaData(); } - //Initializing + // initialization @Override public void init(String[] ids, String dataset, Connection connection) { - super.init(ids, dataset, connection); + super.init(ids, dataset, connection); } - //Your actual Task + // your actual Task @Override public void execute() { } - //Closing the benchmark, freeing some stuff etc. + // closes the benchmark, freeing some stuff etc. @Override public void close() { super.close(); } } - ``` - ## Constructor and Configuration Let's start with the Constructor. The YAML benchmark configuration will provide you the constructor parameters. -Imagine you want to have three different parameters. -The first one should provide an integer (e.g. the time limit of the task) -The second one should provide a list of objects (e.g. a list of integers to use) -The third parameter should provide a map of specific key-value pairs. +Imagine you want to have three different parameters: +- The first one should provide an integer (e.g. the time limit of the task) +- The second one should provide a list of objects (e.g. a list of integers to use) +- The third parameter should provide a map of specific key-value pairs You can set this up by using the following parameters: ```java -public MyBenchmarkTask(Integer param1,List param2,Map param3)throws FileNotFoundException{ - //TODO whatever you need to do with the parameters +public MyBenchmarkTask(Integer param1, List param2, Map param3) throws FileNotFoundException { + // TODO whatever you need to do with the parameters } ``` -Then Your configuration may look like the following +The configuration of your task may then look like the following: ```yaml -... +tasks: className: "MyBenchmarkTask" configuration: param1: 123 param2: - "1" - "2" - param3: - val1: "abc" + param3: + val1: + key: "pair" val2: 123 - ``` -The parameters will then be matched by their names to the names of the parameters of your constructor, allowing multiple constructors - -These are the three types you can represent in a Yaml configuration. -* Single Values -* Lists of Objects -* Key-Value Pairs - +The keys in the configuration file will then be matched against the names of the parameters of your constructor, thus allowing multiple constructors. -## Add Meta Data +## Add Metadata -If you want to add Meta Data to be written in the results file do the following, - -Let noOfWorkers a value you already set. +If you want to add metadata to your results file, implement the following: ```java - /** - * Add extra Meta Data - */ - @Override - public void addMetaData() { - super.addMetaData(); - - Properties extraMeta = new Properties(); - extraMeta.put("noOfWorkers", noOfWorkers); - - //Adding them to the actual meta data - this.metaData.put(COMMON.EXTRA_META_KEY, extraMeta); - } - +/** + * Add extra metadata + */ +@Override +public void addMetaData() { + super.addMetaData(); + + Properties extraMeta = new Properties(); + extraMeta.put("noOfWorkers", noOfWorkers); + + //Adding them to the actual meta data + this.metaData.put(COMMON.EXTRA_META_KEY, extraMeta); +} ``` -Then the resultsfile will contain all the mappings you put in extraMeta. +In this example, we assume `noOfWorkers` is a value you've already set. -## Initialize the Task +Then the results file will contain all the mappings you put in extraMeta. -You may want to initialize your task, set some more values, start something in the background etc. etc. +## Initialize the Task -You will be provided the `suiteID`, `experimentID` and the `taskID` in the `ids` array, as well as the name of the dataset -and the connection currently beeing benchmarked. +In the `init` method, you will be provided with the `suiteID`, `experimentID`, and the `taskID` in the `ids` array, as well as the name of the dataset +and the connection, that is currently being benchmarked. ```java - @Override - public void init(String[] ids, String dataset, Connection connection) { - super.init(ids, dataset, connection); - //ADD YOUR CODE HERE - } +@Override +public void init(String[] ids, String dataset, Connection connection) { + super.init(ids, dataset, connection); + // your initialization code +} ``` -The ids, the dataset and the connection will be set in the `AbstractTask` which you can simply access by using `this.connection` for example. - ## Execute -Now you can create the actual benchmark task you want to use. - +Now you can create the actual benchmark task you want to use: ```java - @Override - public void execute() { - //ADD YOUR CODE HERE - } +@Override +public void execute() { + // ADD YOUR CODE HERE +} ``` -Be aware that if you are using the `workers` implemented in Iguana, you need to stop them after your benchmark using the `worker.stopSending()` method. +Be aware that if you are using the `workers` implemented in Iguana, you have to stop them after your benchmark with the `worker.stopSending()` method. ## Close -If you need to close some streams at the end of your benchmark task, you can do that in the `close` function. +If you need to close resources at the end of your benchmark task, you can do that in the `close` function. Simply override the existing one and call the super method and implement what you need. ```java - @Override - public void close() { - super.close(); - } +@Override +public void close() { + super.close(); + // ... +} ``` ## Full overview @@ -181,45 +166,40 @@ public class MyBenchmarkTask extends AbstractTask { private List param2; private Map param3; - //Your constructor(s) + // your constructor(s) public MyBenchmarkTask(Integer param1, List param2, Map param3) throws FileNotFoundException { - this.param1 = param1; this.param2 = param2; this.param3 = param3; - } - - - //Meta Data (which will be added in the resultsfile) + + // metadata (which will be added in the results file) @Override public void addMetaData() { super.addMetaData(); Properties extraMeta = new Properties(); extraMeta.put("noOfWorkers", noOfWorkers); - - //Adding them to the actual meta data - this.metaData.put(COMMON.EXTRA_META_KEY, extraMeta); + + // adding them to the actual metadata + this.metaData.put(COMMON.EXTRA_META_KEY, extraMeta); } @Override public void init(String[] ids, String dataset, Connection connection) { super.init(ids, dataset, connection); - //ADD YOUR CODE HERE + // ADD YOUR CODE HERE } @Override public void execute() { - //ADD YOUR CODE HERE + // ADD YOUR CODE HERE } - - - //Closing the benchmark, freeing some stuff etc. + + // closing the benchmark, freeing some stuff etc. @Override public void close() { super.close(); } } - ``` diff --git a/docs/develop/extend-workers.md b/docs/develop/extend-workers.md index e900c1922..c8f27f0c3 100644 --- a/docs/develop/extend-workers.md +++ b/docs/develop/extend-workers.md @@ -1,69 +1,66 @@ # Extend Workers -If the implemented workers aren't sufficient you can create your own one. +If the implemented workers aren't sufficient, you can create your own. -Start by extending the `AbstractWorker` +Start by extending the abstract class `AbstractWorker`: ```java package org.benchmark.workers; @Shorthand("MyWorker") public class MyWorker extends AbstractWorker{ - - //Executing the current benchmark query - public void executeQuery(String query, String queryID){ - - } - + + // Executes the current benchmark query + public void executeQuery(String query, String queryID) { + // ... + } } ``` -These are the only two functions you need to implement, the rest is done by the `AbstractWorker`. - -You can override more functions, please consider looking into the javadoc for that. +That is the only function you need to implement. The rest is already done by the `AbstractWorker`, but +you can also override more functions. For that, please consider looking into the javadocs. ## Constructor -The constructor parameters will be provided the same way the Task gets the parameters, thus simply look -at [Extend Task](../extend-task). +The constructor parameters are provided the same way as for the tasks. Thus, simply look at the [Extend Task](../extend-task) page. ## Execute the current query You can execute a query against the current connection (`this.con`). -As this is up to you how to do that, here is an example implementation for using HTTP Get. +This is implementation mainly up to you. Here is an example that uses HTTP GET: ```java @Override - public void executeQuery(String query, String queryID) { - Instant start = Instant.now(); - - try { - String qEncoded = URLEncoder.encode(query, "UTF-8"); - String addChar = "?"; - if (con.getEndpoint().contains("?")) { - addChar = "&"; - } - String url = con.getEndpoint() + addChar + parameter+"=" + qEncoded; - HttpGet request = new HttpGet(url); - RequestConfig requestConfig = RequestConfig.custom().setSocketTimeout(timeOut.intValue()) - .setConnectTimeout(timeOut.intValue()).build(); - - if(this.responseType != null) - request.setHeader(HttpHeaders.ACCEPT, this.responseType); - - request.setConfig(requestConfig); - CloseableHttpClient client = HttpClients.createDefault(); - CloseableHttpResponse response = client.execute(request, getAuthContext(con.getEndpoint())); - - // method to process the result in background - super.processHttpResponse(queryID, start, client, response); - - } catch (Exception e) { - LOGGER.warn("Worker[{{ '{{}}' }} : {{ '{{}}' }}]: Could not execute the following query\n{{ '{{}}' }}\n due to", this.workerType, - this.workerID, query, e); - super.addResults(new QueryExecutionStats(queryID, COMMON.QUERY_UNKNOWN_EXCEPTION, durationInMilliseconds(start, Instant.now()))); +public void executeQuery(String query, String queryID) { + Instant start = Instant.now(); + + try { + String qEncoded = URLEncoder.encode(query, "UTF-8"); + String addChar = "?"; + if (con.getEndpoint().contains("?")) { + addChar = "&"; } + String url = con.getEndpoint() + addChar + parameter+ "=" + qEncoded; + HttpGet request = new HttpGet(url); + RequestConfig requestConfig = RequestConfig.custom().setSocketTimeout(timeOut.intValue()) + .setConnectTimeout(timeOut.intValue()).build(); + + if(this.responseType != null) + request.setHeader(HttpHeaders.ACCEPT, this.responseType); + + request.setConfig(requestConfig); + CloseableHttpClient client = HttpClients.createDefault(); + CloseableHttpResponse response = client.execute(request, getAuthContext(con.getEndpoint())); + + // method to process the result in background + super.processHttpResponse(queryID, start, client, response); + + } catch (Exception e) { + LOGGER.warn("Worker[{{ '{{}}' }} : {{ '{{}}' }}]: Could not execute the following query\n{{ '{{}}' }}\n due to", this.workerType, + this.workerID, query, e); + super.addResults(new QueryExecutionStats(queryID, COMMON.QUERY_UNKNOWN_EXCEPTION, durationInMilliseconds(start, Instant.now()))); } +} ``` diff --git a/docs/develop/maven.md b/docs/develop/maven.md index c030b301f..abd48c530 100644 --- a/docs/develop/maven.md +++ b/docs/develop/maven.md @@ -1,14 +1,11 @@ # Use Iguana as a Maven dependency -Iguana provides 3 packages +Iguana provides 3 packages: +- **iguana.commons** - consists of helper classes +- **iguana.resultprocessor** - consists of the metrics and the result storage workflow +- **iguana.corecontroller** - contains the tasks, workers, query-handler, and the overall benchmarking workflow -**iguana.commons** which consists of some helper classes. - -**iguana.resultprocessor** which consists of metrics and the result storage workflow - -and **iguana.corecontroller** which contains the tasks, the workers, the query handlers, and the overall Iguana workflow - -to use one of these packages in your maven project add the following repository to your pom: +To use one of these packages in your maven project add the following repository to your pom: ```xml diff --git a/docs/develop/overview.md b/docs/develop/overview.md index 3dcb93fd6..50daebe37 100644 --- a/docs/develop/overview.md +++ b/docs/develop/overview.md @@ -1,10 +1,10 @@ # Development Overview -Iguana is open source and available at Github [here](https://github.com/dice-group/Iguana). +Iguana is open source and available on GitHub [here](https://github.com/dice-group/Iguana). There are two main options to work on Iguana. -* Fork the git repository and work directly on Iguana -* or use the [Iguana Maven Packages](https://github.com/orgs/dice-group/packages?repo_name=IGUANA) as a library +1. Fork the git repository and work directly on Iguana +2. Use the [Iguana Maven Packages](https://github.com/orgs/dice-group/packages?repo_name=IGUANA) as a library Iguana is a benchmark framework which can be extended to fit your needs. @@ -12,15 +12,17 @@ Iguana is a benchmark framework which can be extended to fit your needs. There are several things you can extend in Iguana. -* Tasks - Add your benchmark task -* Workers - Your system won't work with HTTP GET or POST, or work completely different? Add your specific worker. -* Query Handling - You do not use Plain Text queries or SPARQL? Add your query handler. -* Language - Want more statistics about your specific queries? The result size isn't accurate? add your language support -* Result Storage - Don't want to use RDF? Add your own solution to store the benchmark results. -* Metrics - The metrics won't fit your needs? Add your own. +| Module | Description | +|----------------|----------------------------------------------------------------------------------------------------------------------| +| Tasks | Add your own benchmark task . | +| Workers | Your system won't work with HTTP GET or POST, or works completely different? Add your specific worker. | +| Query Handling | You do not use Plain Text queries or SPARQL? Add your query handler. | +| Language | You want more statistics about your specific queries? The result size isn't accurate? Add support for your language. | | +| Result Storage | You don't want to use RDF? Add your own solution to store the benchmark results. | +| Metrics | The metrics won't fit your needs? Add your own. | ## Bugs -For bugs please open an issue at our [Github Issue Tracker](https://github.com/dice-group/Iguana/issues) +If you find bugs, please open an issue at our [Github Issue Tracker](https://github.com/dice-group/Iguana/issues). diff --git a/docs/download.md b/docs/download.md index 08d8a3390..b25787243 100644 --- a/docs/download.md +++ b/docs/download.md @@ -5,19 +5,19 @@ You need to have Java 11 or higher installed. -In Ubuntu you can do this by +In Ubuntu, you can install it by executing the following command: ```bash sudo apt-get install java ``` ## Download -Please download the latest release at [https://github.com/dice-group/IGUANA/releases/latest](https://github.com/dice-group/IGUANA/releases/latest) +Please download the latest release from [here](https://github.com/dice-group/IGUANA/releases/latest). -The zip file contains 3 files. +The zip file contains 3 files: -* iguana-{{ release_version }}.jar -* example-suite.yml -* start-iguana.sh +* `iguana-{{ release_version }}.jar` +* `example-suite.yml` +* `start-iguana.sh` -The example-suite.yml is a valid benchmark configuration which you can adjust to your needs using the [Configuration](../usage/configuration) wiki. +The `example-suite.yml` is a valid benchmark configuration that you can adjust to your needs using the [Configuration](../usage/configuration) wiki. diff --git a/docs/index.md b/docs/index.md index 06543a855..a4b4e1556 100644 --- a/docs/index.md +++ b/docs/index.md @@ -2,23 +2,23 @@ IGUANA Logo -This documentation will help you benchmark your HTTP endpoints (such as your Triple store) using Iguana and help you extend Iguana to your needs. -It is split into three parts +This documentation will help you benchmark your HTTP endpoints (such as your triple store) using Iguana and help you extend Iguana to your needs. +It is split into four parts: * General * Quick Start Guide * Usage * Development -In **General** you will find a bit of information of what Iguana is and what it's capable of. +In **General** you will find general information about Iguana and what it's capable of. -In the **Quick Start Guide** you will find how to download and start Iguana as well how to quickly configure your first simple benchmark using Iguana. +In the **Quick Start Guide** you will learn how to download and start Iguana, as well as how to quickly configure your first simple benchmark using Iguana. -In **Usage** you will find everything on how to execute a benchmark with Iguana and how to configure the benchmark to your needs. +In **Usage** you will learn everything on how to execute a benchmark with Iguana and how to configure the benchmark to your needs. It further provides details on what tests Iguana is capable of. -A Tutorial will finally guide you through all steps broadly which you can use as a quick start. +A Tutorial will finally guide you through every step broadly.You can also use this page as a quick start. -In **Development** you will find everything you need to know in case that Iguana isn't sufficient for your needs. It shows how to extend Iguana to use your metrics or your specific benchmark test +In **Development** you will find everything you need to know in case Iguana isn't sufficient for your needs. It shows you how to extend Iguana with missing metrics or your specific benchmarking test. diff --git a/docs/quick-config.md b/docs/quick-config.md index f45f99976..3d6b447a7 100644 --- a/docs/quick-config.md +++ b/docs/quick-config.md @@ -1,12 +1,12 @@ # Quickly Configure Iguana -Here we will setup a quick configuration which will benchmark one triple store (e.g. apache jena fuseki) using one simulated user. +Here we will set up a quick configuration that will benchmark a triple store (e.g. apache jena fuseki) using one simulated user. We assume that your triple store (or whatever HTTP GET endpoint you want to use) is running and loaded with data. -For now we assume that the endpoint is at `http://localhost:3030/ds/sparql` and uses GET with the parameter `query` +For now, we assume that the endpoint is at `http://localhost:3030/ds/sparql` and uses GET with the parameter `query`. -Further on the benchmark should take 10 minutes (or 60.000 ms) and uses plain text queries located in `queries.txt`. +Further on the benchmark should take 10 minutes (or 600,000 ms) and uses plain text queries located in `queries.txt`. -If you do not have created some queries yet, use these for example +If you do not have created some queries yet, you can use the following examples by saving them to the file `queries.txt` in the same directory as the executable: ```sparql SELECT * {?s ?p ?o} @@ -14,12 +14,10 @@ SELECT * {?s ?p ?o} LIMIT 10 SELECT * {?s ?o} ``` -and save them to `queries.txt`. +Your results will be written as an N-Triple file to `first-benchmark-results.nt`. -Your results will be written as an N-Triple file to `first-benchmark-results.nt` - -The following configuration works with these demands. +The following configuration works with this setup: ```yaml # you can ignore this for now @@ -38,9 +36,7 @@ tasks: configuration: # 10 minutes (time Limit is in ms) timeLimit: 600000 - - # create one SPARQL Worker (it's basically a HTTP get worker using the 'query' parameter - # it uses the queries.txt file as benchmark queries + workers: - threads: 1 className: "HttpGetWorker" @@ -56,4 +52,4 @@ storages: ``` -For more information on the confguration have a look at [Configuration](../usage/configuration/) +For more information on the configuration, have a look at [Configuration](../usage/configuration/) diff --git a/docs/run-iguana.md b/docs/run-iguana.md index 809f5f533..486c8608f 100644 --- a/docs/run-iguana.md +++ b/docs/run-iguana.md @@ -1,26 +1,20 @@ # Start a Benchmark -Start Iguana with a benchmark suite (e.g the example-suite.yml) either using the start script +Start Iguana with a benchmark suite (e.g. the example-suite.yml) either by using the start script: ```bash ./start-iguana.sh example-suite.yml ``` -To set JVM options, you can use `$IGUANA_JVM` +or by directly executing the jar-file: -For example to let Iguana use 4GB of RAM you can set the `IGUANA_JVM` as follows ```bash -export IGUANA_JVM=-Xmx4g +java -jar iguana-{{ release_version }}.jar example-suite.yml ``` -and start as above. - - - -or using the jar with java 11 as follows - +To set JVM options, if you're using the script, you can set the environment variable `$IGUANA_JVM`. +For example, to let Iguana use 4GB of RAM you can set `IGUANA_JVM` as follows: ```bash -java -jar iguana-corecontroller-{{ release_version }}.jar example-suite.yml +export IGUANA_JVM=-Xmx4g ``` - diff --git a/docs/shorthand-mapping.md b/docs/shorthand-mapping.md index b4e43b815..49cdd8647 100644 --- a/docs/shorthand-mapping.md +++ b/docs/shorthand-mapping.md @@ -1,27 +1,27 @@ -| Shorthand | Class Name | -|----------|-------| -| Stresstest | `org.aksw.iguana.cc.tasks.impl.Stresstest` | -|----------|-------| -| lang.RDF | `org.aksw.iguana.cc.lang.impl.RDFLanguageProcessor` | -| lang.SPARQL | `org.aksw.iguana.cc.lang.impl.SPARQLLanguageProcessor` | -| lang.SIMPLE | `org.aksw.iguana.cc.lang.impl.ThrowawayLanguageProcessor` | -|----------|-------| -| UPDATEWorker | `org.aksw.iguana.cc.worker.impl.UPDATEWorker` | -| HttpPostWorker | `org.aksw.iguana.cc.worker.impl.HttpPostWorker` | -| HttpGetWorker | `org.aksw.iguana.cc.worker.impl.HttpGetWorker` | -| CLIWorker | `org.aksw.iguana.cc.worker.impl.CLIWorker` | -| CLIInputWorker | `org.aksw.iguana.cc.worker.impl.CLIInputWorker` | -| CLIInputFileWorker | `org.aksw.iguana.cc.worker.impl.CLIInputFileWorker` | -| CLIInputPrefixWorker | `org.aksw.iguana.cc.worker.impl.CLIInputPrefixWorker` | -| MultipleCLIInputWorker | `org.aksw.iguana.cc.worker.impl.MultipleCLIInputWorker` | -|----------|-------| -| NTFileStorage | `org.aksw.iguana.rp.storages.impl.NTFileStorage` | -| RDFFileStorage | `org.aksw.iguana.rp.storages.impl.RDFFileStorage` | -| TriplestoreStorage | `org.aksw.iguana.rp.storages.impl.TriplestoreStorage` | -|----------|-------| -| QPS | `org.aksw.iguana.rp.metrics.impl.QPSMetric` | -| AvgQPS | `org.aksw.iguana.rp.metrics.impl.AvgQPSMetric` | -| NoQ | `org.aksw.iguana.rp.metrics.impl.NoQMetric` | -| NoQPH | `org.aksw.iguana.rp.metrics.impl.NoQPHMetric` | -| QMPH | `org.aksw.iguana.rp.metrics.impl.QMPHMetric` | -| EachQuery | `org.aksw.iguana.rp.metrics.impl.EQEMetric` | +| Shorthand | Class Name | +|------------------------|-----------------------------------------------------------| +| Stresstest | `org.aksw.iguana.cc.tasks.impl.Stresstest` | +| ---------- | ------- | +| lang.RDF | `org.aksw.iguana.cc.lang.impl.RDFLanguageProcessor` | +| lang.SPARQL | `org.aksw.iguana.cc.lang.impl.SPARQLLanguageProcessor` | +| lang.SIMPLE | `org.aksw.iguana.cc.lang.impl.ThrowawayLanguageProcessor` | +| ---------- | ------- | +| UPDATEWorker | `org.aksw.iguana.cc.worker.impl.UPDATEWorker` | +| HttpPostWorker | `org.aksw.iguana.cc.worker.impl.HttpPostWorker` | +| HttpGetWorker | `org.aksw.iguana.cc.worker.impl.HttpGetWorker` | +| CLIWorker | `org.aksw.iguana.cc.worker.impl.CLIWorker` | +| CLIInputWorker | `org.aksw.iguana.cc.worker.impl.CLIInputWorker` | +| CLIInputFileWorker | `org.aksw.iguana.cc.worker.impl.CLIInputFileWorker` | +| CLIInputPrefixWorker | `org.aksw.iguana.cc.worker.impl.CLIInputPrefixWorker` | +| MultipleCLIInputWorker | `org.aksw.iguana.cc.worker.impl.MultipleCLIInputWorker` | +| ---------- | ------- | +| NTFileStorage | `org.aksw.iguana.rp.storages.impl.NTFileStorage` | +| RDFFileStorage | `org.aksw.iguana.rp.storages.impl.RDFFileStorage` | +| TriplestoreStorage | `org.aksw.iguana.rp.storages.impl.TriplestoreStorage` | +| ---------- | ------- | +| QPS | `org.aksw.iguana.rp.metrics.impl.QPSMetric` | +| AvgQPS | `org.aksw.iguana.rp.metrics.impl.AvgQPSMetric` | +| NoQ | `org.aksw.iguana.rp.metrics.impl.NoQMetric` | +| NoQPH | `org.aksw.iguana.rp.metrics.impl.NoQPHMetric` | +| QMPH | `org.aksw.iguana.rp.metrics.impl.QMPHMetric` | +| EachQuery | `org.aksw.iguana.rp.metrics.impl.EQEMetric` | diff --git a/docs/usage/configuration.md b/docs/usage/configuration.md index 8568a4263..851a76802 100644 --- a/docs/usage/configuration.md +++ b/docs/usage/configuration.md @@ -1,6 +1,7 @@ # Configuration -The Configuration explains Iguana how to execute your benchmark. It is divided into 5 categories +The configuration tells Iguana how it should execute your benchmark. +It is divided into five categories: * Connections * Datasets @@ -8,26 +9,27 @@ The Configuration explains Iguana how to execute your benchmark. It is divided i * Storages * Metrics -Additionally a pre and post task script hook can be set. +Additionally, a pre- and post-task script hook can be set. -The configuration has to be either in YAML or JSON. Each section will be detailed out and shows configuration examples. At the end the full configuration will be shown. -For this we will stick to the YAML format, however the equivalent JSON is also valid and can be parsed by Iguana. +The configuration has to be either written in YAML or JSON. Each section contains detailed information and shows configuration examples. +In the end, the full configuration example will be shown. +For this documentation, we will stick to the YAML format, however, the equivalent JSON format can be parsed by Iguana too. ### Connections -Every benchmark suite can execute several connections (e.g. an HTTP endpoint, or a CLI application). -A connection has the following items +Every benchmark suite can execute tasks on several connections (e.g. an HTTP endpoint, or a CLI application). +A connection has the following items: -* name - the name you want to give the connection, which will be saved in the results. -* endpoint - the HTTP endpoint or CLI call. -* updateEndpoint - If your HTTP endpoint is an HTTP Post endpoint set this to the post endpoint. (optional) -* user - for authentication purposes (optional) -* password - for authentication purposes (optional) -* version - setting the version of the tested triplestore, if set resource URI will be ires:name-version (optional) +* `name` - the name you want the connection to have, the name will be saved in the results +* `endpoint` - the HTTP endpoint or CLI call +* `updateEndpoint` - if your HTTP endpoint is an HTTP POST endpoint, you can set it with this item (optional) +* `user` - for authentication purposes (optional) +* `password` - for authentication purposes (optional) +* `version` - sets the version of the tested triplestore; if this is set, the resource URI will be ires:name-version (optional) -To setup an endpoint as well as an updateEndpoint might be confusing at first, but if you to test read and write performance simultanously and how updates might have an impact on read performance, you can set up both. +At first, it might be confusing to set up both an `endpoint` and `updateEndpoint`, but it is used, when you want your test to perform read and write operations simultaneously, for example, to test the impact of updates on the read performance of your triple store. -For more detail on how to setup the CLI call look at [Implemented Workers](../workers). There are all CLI Workers explained and how to set the endpoint such that the application will be run correctly. +For more detail on how to set up the CLI call look at [Implemented Workers](../workers). There, all CLI Workers will be explained and how to properly set them up. Let's look at an example: @@ -43,22 +45,21 @@ connections: password: "secret" ``` -Here we have two connections: System1 and System2. System1 is only setup to use an HTTP Get endpoint at http://localhost:8800/query. System2 however uses authentication and has an update endpoint as well, and thus will be correctly test with updates (POSTs) too. +Here we have two connections: System1 and System2. System1 is only set up to use an HTTP GET endpoint at http://localhost:8800/query. System2, however, uses authentication and has an update endpoint, and thus will be correctly tested with updates (POSTs) too. ### Datasets -Pretty straight forward. You might want to test your system with different datasets (e.g. databases, triplestores etc.) -If you system does not work on different datasets, just add one datasetname like +You might want to test your system with different datasets (e.g. databases, triple stores). +If your system does not work on different datasets, just add a single dataset-name like this: ```yaml datasets: - name: "DoesNotMatter" ``` -otherwise you might want to benchmark different datasets. Hence you can setup a Dataset Name, as well as file. -The dataset name will be added to the results, whereas both can be used in the task script hooks, to automatize dataset load into your system. +Otherwise, if you're using multiple different datasets, you can set a `name` and a `file` for each dataset. Both items can later be used in the pre- and post-task scripts to automate the loading of data into your system. The name is also used to distinguish the datasets in the result of the benchmark. -Let's look at an example: +A configuration with multiple datasets may look like this: ```yaml datasets: @@ -69,13 +70,12 @@ datasets: ### Tasks -A Task is one benchmark Task which will be executed against all connections for all datasets. -A Task might be a stresstest which we will be using in this example. Have a look at the full configuration of the [Stresstest](../stresstest#Configuration) +A Task is a benchmark task which will be executed against all connections for all datasets. A task might be, for example, the included [Stresstest](../stresstest#Configuration). -The configuration of one Task consists of the following: +The configuration of a task consists of the following keys: -* className - The className or [Shorthand](#Shorthand) -* configuration - The parameters of the task +* `className` - The classname of the task or its [Shorthand](#Shorthand) +* `configuration` - The parameters for the task ```yaml tasks: @@ -85,7 +85,7 @@ tasks: parameter2: "value2" ``` -Let's look at an example: +The following shows an exemplary configuration for the `tasks` key: ```yaml tasks: @@ -110,31 +110,34 @@ tasks: timeOut: 180000 ``` -We configured two Tasks, both Stresstests. The first one will be executed for one hour and uses simple text queries -which can be executed right away. -Further on it uses 2 simulated HttpGetWorkers with the same configuration. -At this point it's recommend to check out the [Stresstest Configuration](../stresstest#Configuration) in detail for -further configuration. +In this configuration we have two tasks of the included Stresstest. + +The first task has two workers of the class `HttpGetWorkers`, that execute the given queries simultaneously and independently of each other for an hour. + +The second task has also two workers of the class `HttpGetWorkers`, but they will only execute every given query once. + +For further details, check out the [Stresstest configuration](../stresstest#Configuration) page. ### Storages -Tells Iguana how to save your results. Currently Iguana supports two solutions +The `storages` setting will tell Iguana how it should save your results. Currently Iguana supports three solutions: -* NTFileStorage - will save your results into one NTriple File. -* RDFFileStorage - will save your results into an RDF File (default TURTLE). -* TriplestoreStorage - Will upload the results into a specified Triplestore +* NTFileStorage - saves your results as an NTriple File. +* RDFFileStorage - saves your results as an RDF File (default TURTLE). +* TriplestoreStorage - uploads the results into a specified triple store -This is optional. The default storage is `NTFileStorage`. +This setting is optional. The default storage is `NTFileStorage`. -**NTFileStorage** can be setup by just stating to use it like +#### **NTFileStorage** +You can set the NTFileStorage solution with the following configuration: ```yaml storages: - className: "NTFileStorage" ``` -However it can be configured to use a different result file name. The default is `results_{DD}-{MM}-{YYYY}_{HH}-{mm}.nt`. -See example below. +However, it can also be configured to use a different result file name. The default file name is `results_{DD}-{MM}-{YYYY}_{HH}-{mm}.nt`. +See the example below: ```yaml storages: @@ -143,35 +146,34 @@ storages: configuration: fileName: "results-of-my-benchmark.nt" ``` - -The **RDFFileStorage** is similar to the NTFileStorage but will determine the RDF format from the file extension -To use RDF/XML f.e. you would end the file on .rdf, for TURTLE end it on .ttl +#### RDFFileStorage +The **RDFFileStorage** is similar to the NTFileStorage, but it will determine the RDF format from the given file extension. +To use RDF/XML you would end the file name with the `.rdf` extension, for TURTLE end it with the `.ttl` extension. ```yaml storages: - - className: "NTFileStorage" + - className: "RDFFileStorage" #optional configuration: - fileName: "results-of-my-benchmark.rdf" + fileName: "results-of-my-benchmark.ttl" ``` - - +#### TriplestoreStorage The **TriplestoreStorage** can be configured as follows: ```yaml storages: - - className: TriplestoreStorage + - className: "TriplestoreStorage" configuration: endpoint: "http://localhost:9999/sparql" updateEndpoint: "http://localhost:9999/update" ``` -if you triple store uses authentication you can set that up as follows: +If your triple store uses authentication, you can set it up as follows: ```yaml storages: - - className: TriplestoreStorage + - className: "TriplestoreStorage" configuration: endpoint: "http://localhost:9999/sparql" updateEndpoint: "http://localhost:9999/update" @@ -179,25 +181,24 @@ storages: password: "secret" ``` - -For further detail on how to read the results have a look [here](../results) - - +For further detail on how to read the results, have a look [here](../results). ### Metrics -Let's Iguana know what Metrics you want to include in the results. +The `metrics` setting lets Iguana know what metrics you want to include in the results. Iguana supports the following metrics: -* Queries Per Second (QPS) -* Average Queries Per Second (AvgQPS) -* Query Mixes Per Hour (QMPH) -* Number of Queries successfully executed (NoQ) -* Number of Queries per Hour (NoQPH) -* Each query execution (EachQuery) - experimental +* Queries Per Second (`QPS`) +* Average Queries Per Second (`AvgQPS`) +* Query Mixes Per Hour (`QMPH`) +* Number of Queries successfully executed (`NoQ`) +* Number of Queries per Hour (`NoQPH`) +* Each query execution (`EachQuery`) - experimental + +For more details on each of the metrics have a look at the [Metrics](../metrics) page. -For more detail on each of the metrics have a look at [Metrics](../metrics) +The `metrics` setting is optional and the default is set to every available metric, except `EachQuery`. Let's look at an example: @@ -210,8 +211,9 @@ metrics: - className: "NoQPH" ``` -In this case we use all the default metrics which would be included if you do not specify `metrics` in the configuration at all. -However you can also just use a subset of these like the following: +In this case we use every metric that Iguana has implemented. This is the default. + +However, you can also just use a subset of these metrics: ```yaml metrics: @@ -219,34 +221,37 @@ metrics: - className: "AvgQPS" ``` -For more detail on how the results will include these metrics have a look at [Results](../results). +For more details on how the results will include these metrics, have a look at [Results](../results). ### Task script hooks -To automatize the whole benchmark workflow, you can setup a script which will be executed before each task, as well as a script which will be executed after each task. +To automate the whole benchmark workflow, you can optionally set up a script which will be executed before each task, as well as a script which will be executed after each task. -To make it easier, the script can get the following values +You can have different scripts for different datasets, connections or tasks, by using the following variables in the `preScriptHook` and `postScriptHook` +setting: -* dataset.name - The current dataset name -* dataset.file - The current dataset file name if there is anyone -* connection - The current connection name -* connection.version - The current connection version, if no version is set -> {{ '{{connection.version}}' }} -* taskID - The current taskID +* `dataset.name` - The name of the current dataset this task is executed with +* `dataset.file` - The file of the current dataset +* `connection` - The name of the current connection this task is executed with +* `connection.version` - The version of the current connection +* `taskID` - The current taskID +You can use these variables by using brackets like this: +`{{connection}}`. -You can set each one of them as an argument using brackets like `{{ '{{connection}}' }}`. -Thus you can setup scripts which will start your system and load it with the correct dataset file beforehand and stop the system after every task. +Iguana will then instantiate these variables with the appropriate values and execute the values of `preScriptHook` and `postScriptHook`. -However these script hooks are completely optional. - -Let's look at an example: +This is what a full example could look like: ```yaml -preScriptHook: "/full/path/{{ '{{connection}}' }}-{{ '{{connection.version}}' }}/load-and-start.sh {{ '{{dataset.file}}' }}" -postScriptHook: "/full/path/{{ '{{connection}}' }}/stop.sh" - + preScriptHook: "/full/path/{{connection}}-{{connection.version}}/load-and-start.sh {{dataset.file}}" + postScriptHook: "/full/path/{{connection}}/stop.sh" ``` +With this, you can set up scripts which will start, and load your system with the correct datasets before the task execution, and stop the system after the execution. + +For a full example, see the [Tutorial](../tutorial) page. + ### Full Example ```yaml @@ -285,8 +290,8 @@ tasks: location: "queries.txt" timeOut: 180000 -preScriptHook: "/full/path/{{ '{{connection}}' }}/load-and-start.sh {{ '{{dataset.file}}' }}" -postScriptHook: "/full/path/{{ '{{connection}}' }}/stop.sh" +preScriptHook: "/full/path/{{connection}}/load-and-start.sh {{dataset.file}}" +postScriptHook: "/full/path/{{connection}}/stop.sh" metrics: @@ -306,8 +311,8 @@ storages: ### Shorthand -A shorthand is a short name for a class in Iguana which can be used in the configuration instead of the complete class name: -e.g. instead of +A shorthand is a short name for a class in Iguana which can be used in the configuration instead of the complete class name. +For example, instead of: ```yaml storages: diff --git a/docs/usage/getting-started.md b/docs/usage/getting-started.md index d281333b8..dcc56e086 100644 --- a/docs/usage/getting-started.md +++ b/docs/usage/getting-started.md @@ -1,49 +1,63 @@ ## What is Iguana -Iguana is a HTTP and CLI read/write performance benchmark framework suite. -It can stresstest HTTP get and post endpoints as well as CLI applications using a bunch of simulated users which will bombard the endpoint using queries. -Queries can be anything. SPARQL, SQL, Text and anything else you can fit in one line. +Iguana is an HTTP and CLI read/write performance benchmark framework suite. +It can stresstest HTTP get and post endpoints as well as CLI applications using a bunch of simulated users which will flood the endpoint using queries. +Queries can be anything. SPARQL, SQL, Text, etc. ### What can be benchmarked -Iguana is capable of benchmarking and stresstesting the following applications +Iguana is capable of benchmarking and stresstesting the following applications: -* HTTP GET and POST endpoint (e.g. Triple Stores, REST Services, Question Answering endpoints) +* HTTP GET and POST endpoints (e.g. Triple Stores, REST Services, Question Answering endpoints) * CLI Applications which either * exit after every query - * or awaiting input after each query + * await for input after each query ### What Benchmarks are possible Every simulated User (named Worker in the following) gets a set of queries. -These queries have to be saved in one file, whereas each query is one line. -Hence, everything you can fit in one line (e.g a SPARQL query, a text question, an RDF document) can be used as a query -and a set of these queries represent the benchmark. +These queries can be saved in a file or in a folder. +Hence, everything you can fit in one line (e.g a SPARQL query, a text question, an RDF document) can be used as a query and a set of these queries represent the benchmark. Iguana will then let every Worker execute these queries against the endpoint. +## Prerequisites + +You need to have Java 11 or higher installed. + +In Ubuntu you can install it by executing the following command: +```bash +sudo apt-get install java +``` + ## Download -Please download the latest release at [https://github.com/dice-group/IGUANA/releases/latest](https://github.com/dice-group/IGUANA/releases/latest) +Please download the latest release from [here](https://github.com/dice-group/IGUANA/releases/latest). -The zip file contains 3 files. +The zip file contains 3 files: -* iguana-corecontroller-x.y.z.jar -* example-suite.yml -* start.sh +* `iguana-{{ release_version }}.jar` +* `example-suite.yml` +* `start-iguana.sh` -The example-suite.yml is a valid benchmark configuration which you can adjust to your needs using the [Configuration](Configuration) wiki. +The `example-suite.yml` is a valid benchmark configuration that you can adjust to your needs using the [Configuration](../configuration) wiki. ## Start a Benchmark -Start Iguana with a benchmark suite (e.g the example-suite.yml) either using the start script +Start Iguana with a benchmark suite (e.g. the `example-suite.yml`) either by using the start script: ```bash ./start-iguana.sh example-suite.yml ``` -or using java 11 if you want to give Iguana more RAM or in general set JVM options. +or by directly executing the jar-file: ```bash -java -jar iguana-corecontroller-4.0.0.jar example-suite.yml +java -jar iguana-{{ release_version }}.jar example-suite.yml ``` +To set JVM options, if you're using the script, you can set the environment variable `$IGUANA_JVM`. + +For example, to let Iguana use 4GB of RAM you can set `IGUANA_JVM` as follows: +```bash +export IGUANA_JVM=-Xmx4g +``` diff --git a/docs/usage/languages.md b/docs/usage/languages.md index 18af1e166..85c4796cf 100644 --- a/docs/usage/languages.md +++ b/docs/usage/languages.md @@ -1,15 +1,15 @@ # Supported Languages -The Language tag is set to assure that the result size returned by the benchmarked system is correctly read and that result can give a little extra query statistics. +The Language tag assures that the size of the result of each query, returned by the benchmarked system, is read correctly and that the result can give some extra statistics about the query. -Currently two languages are implemented, however you can use `lang.SPARQL` or simply ignore it all the way. +Currently, two languages are implemented, however you can use `lang.SPARQL` or simply ignore it all the way. If they are not in `SPARQL` the query statistics will be just containing the query text and the result size will be read as if each returned line were one result. -Additionaly a SIMPLE language tag is added which parses nothing and sets the result size as the content length of the results. +Additionally, a `lang.SIMPLE` tag is added which parses nothing and sets the result size as the content length of the results. -If you work with results which have a content length >=2GB please use `lang.SIMPLE`, as `lang.SPARQL` and `lang.RDF` cannot work with results >=2GB at the moment. +If you work with results that have a content length >=2GB please use `lang.SIMPLE`, as `lang.SPARQL` and `lang.RDF` cannot work with results >=2GB at the moment. -The 3 languages are: +The 3 supported languages are: * `lang.SPARQL` * `lang.RDF` diff --git a/docs/usage/metrics.md b/docs/usage/metrics.md index ff5abc4af..5d832a6f1 100644 --- a/docs/usage/metrics.md +++ b/docs/usage/metrics.md @@ -1,7 +1,7 @@ # Implemented Metrics -Every metric will be calculated globally (for one Experiment Task) and locally (for each Worker) -Hence you can just analyze the overall metrics or if you want to look closer, you can look at each worker. +Every metric will be calculated globally (for one Experiment Task) and locally (for each Worker). +Hence, you are able to analyze the metrics of the whole benchmark or only of each worker. ## NoQ @@ -9,40 +9,40 @@ The number of successfully executed Queries ## QMPH -The number of executed Query Mixes Per Hour +The number of executed Query Mixes Per Hour ## NoQPH -The number of successfully executed Number of Queries Per Hour +The number of successfully executed Queries Per Hour ## QPS -For each query the `queries per second`, the `total time` in ms (summed up time of each execution), the no of `succeeded` and `failed` executions and the `result size` will be saved. -Additionaly will try to tell how many times a query failed with what reason. (`timeout`, `wrong return code` e.g. 400, or `unknown`) +For each query, the `queries per second`, the `total time` in milliseconds (summed up time of each execution), the number of `succeeded` and `failed` executions, and the `result size` will be saved. -Further on the QPS metrics provides a penalized QPS which penalizes queries which will fail. -As some systems who cannot resolve a query just returns an error code and thus can have a very high score, even though they could only handle a few queries it would be rather unfair to the compared systems. Thus we introduced the penalty QPS. It is calculated the same as the QPS score, but for each failed query it uses the penalty instead of the actual time the failed query took. +Additionally, Iguana will try to tell you how many times a query has failed and for what reason (`timeout`, `wrong return code`, e.g. 400, or `unknown`). -The default is set to the timeOut of the task. -However you can override it as follows: +Further on the QPS metric provides a penalized QPS metric that penalizes queries that fail. +Some systems just return an error code, if they can't resolve a query, thus they can have a very high score, even though they were only able to handle a few queries. That would be rather unfair to the compared systems, therefore we introduced the penalty QPS. It is calculated the same as the QPS score, but for each failed query it uses the penalty instead of the actual time the failed query took. + +The default penalty is set to the `timeOut` value of the task. However, you can override it as follows: ```yaml metrics: - className: "QPS" configuration: #in MS - penality: 10000 + penalty: 10000 ``` ## AvgQPS The average of all queries per second. -Also adding a penalizedAvgQPS. Default penalty is timeOut, can be overwritten as follows: +It also adds a penalizedAvgQPS metric. The default penalty is set to the `timeOut` value of the task, but it can be overwritten as follows: ```yaml metrics: - className: "AvgQPS" - confiugration: + configuration: # in ms penalty: 10000 ``` diff --git a/docs/usage/queries.md b/docs/usage/queries.md index 65508aab0..6b2dda26a 100644 --- a/docs/usage/queries.md +++ b/docs/usage/queries.md @@ -5,12 +5,12 @@ The following parameters can be set: | parameter | optional | default | description | |-----------|----------|----------------|--------------------------------------------------------------------------------------------------------------------------------------| -| location | no | | The location of the queries | -| format | yes | "one-per-line" | Format how the queries are stored in the file(s) (see [Query Format](#query-format)) | -| caching | yes | true | Indicating if the queries should be loaded into memory or read from file when needed (see [Caching](#caching)) | +| location | no | | The file path to the queries | +| format | yes | "one-per-line" | The format of how the queries are stored in the file(s) (see [Query Format](#query-format)) | +| caching | yes | true | Indicates if the queries should be loaded into the memory or read from a file everytime they are needed (see [Caching](#caching)) | | order | yes | "linear" | The order in which the queries are read from the source (see [Query Order](#query-order)) | | pattern | yes | | The configuration to be used to generate [SPARQL Pattern Queries](#sparql-pattern-queries) | -| lang | yes | "lang.SPARQL" | The language the queries and response are in (e.g. SPARQL). Basically just creates some more statistics (see [Langauge](#language) ) | +| lang | yes | "lang.SPARQL" | The language the queries and responses are in (e.g. SPARQL). Basically just creates some more statistics (see [Langauge](#language)) | For example: @@ -33,12 +33,12 @@ described in [SPARQL Pattern Queries](#sparql-pattern-queries). ## Query Format -A query can be anything: SPARQL, SQL, a whole book if you need to. +A query can be anything: SPARQL, SQL, or a whole book if you need to. The queries can be provided in different formats: - one file with: - one query per line - - multi line queries, separated by a separator line + - multi-line queries, separated by a separator line - a folder with query files; one query per file The format is configured using the `format` parameter. @@ -50,7 +50,7 @@ The configuration for this format is: ```yaml queries: - location: "path/to/queries" + location: "path/to/file" format: "one-per-line" ``` @@ -74,11 +74,11 @@ The configuration for this format is: ```yaml queries: - location: "path/to/queries" + location: "path/to/file" format: "separator" ``` -However, you can set the separator in the configuration. +However, you can also set the separator line in the configuration. For example if the separator is an empty line, the file can look like this: ``` @@ -95,7 +95,7 @@ The configuration for this format is: ```yaml queries: - location: "path/to/queries" + location: "path/to/file" format: separator: "" ``` @@ -108,7 +108,7 @@ The configuration for this format is: ```yaml queries: - location: "path/to/queries" + location: "path/to/folder" format: "folder" ``` @@ -116,7 +116,7 @@ queries: If the `caching` parameter is set to `true`, the queries are loaded into memory when the worker is initialized. This is the **default**. -If the `caching` parameter is set to `false`, the queries are read from file when needed. This is useful if the queries +If the `caching` parameter is set to `false`, the queries are read from a file every time they are needed. This is useful if the queries are very large, and you don't want all of them to be in memory at the same time. An example configuration is: @@ -145,7 +145,7 @@ queries: ### Random Order The queries are read in a (pseudo) random order. -The if no explicit seed is given, the generated workerID is used as seed, to ensure that each worker starts at the same +If no explicit seed is given, the generated workerID is used as the seed, to ensure that each worker starts with the same query each time. The configuration is: @@ -183,21 +183,21 @@ Hence, Iguana can create thousands of queries using a SPARQL pattern query. A pattern query might look like the following: ```sparql -SELECT * {?s rdf:type %%var0%% ; %%var1%% %%var2%%. %%var2%% ?p ?o} +PREFIX rdf: SELECT * {?s rdf:type %%var0%% ; %%var1%% %%var2%%. %%var2%% ?p ?o} ``` -This query in itself cannot be sent to a triple store, however we can exchange the variables using real data. +This query in itself cannot be sent to a triple store, however, we can exchange the variables using real data. Thus, we need a reference endpoint (ideally) containing the same data as the dataset which will be tested. -This query will then be exchanged to +This query will then be exchanged to: ```sparql -SELECT ?var0 ?var1 ?var2 {?s rdf:type ?var0 ; ?var1 ?var2. ?var2 ?p ?o} LIMIT 2000 +PREFIX rdf: SELECT ?var0 ?var1 ?var2 {?s rdf:type ?var0 ; ?var1 ?var2. ?var2 ?p ?o} LIMIT 2000 ``` and be queried against the reference endpoint. -For each result (limited to 2000) a query instance will be created. +For each result (limited to 2000 by default), a query instance will be created. This will be done for every query in the benchmark queries. All instances of these query patterns will be subsumed as if they were one query in the results. @@ -220,4 +220,4 @@ queries: outputFolder: "queryCache" ``` -If the `outputFolder` contains a fitting cache file, the queries will not be generated again. +If the `outputFolder` already contains a fitting cache file, the queries will not be generated again. diff --git a/docs/usage/results.md b/docs/usage/results.md index 0b77ca3b9..837672a61 100644 --- a/docs/usage/results.md +++ b/docs/usage/results.md @@ -7,7 +7,7 @@ The results are saved into RDF. For those who don't know what RDF is, it is best described as a way to represent a directed graph. The according query language is called SPARQL. -The graph schema of an iguana result is shown above, where as each node represents a class object containg several annotations. +The graph schema of an Iguana result is shown above, where each node represents a class object containing several annotations. To retrieve all TaskIDs you can do the following: @@ -35,7 +35,7 @@ PREFIX iont: PREFIX ires: SELECT ?noq { - ires:123/1/1 iprop:NoQ ?noq + ires:123/1/1 iprop:NoQ ?noq . } ``` @@ -48,12 +48,12 @@ PREFIX iont: PREFIX ires: SELECT ?workerID ?noq { - ires:123/1/1 iprop:workerResult ?workerID - ?workerID iprop:NoQ ?noq + ires:123/1/1 iprop:workerResult ?workerID . + ?workerID iprop:NoQ ?noq . } ``` -However if you just want to see the global NoQ metric for all taskIDs in your results do the following: +However, if you just want to see the global NoQ metric for all taskIDs in your results do the following: ```sparql PREFIX rdf: @@ -87,8 +87,8 @@ SELECT ?executedQuery ?qps ?failed ?resultSize { ?executedQuery iprop:resultSize ?resultSize . } ``` -This will get you the QPS value, the no. of failed queries and the result size of the query. +This will get you the QPS value, the number of failed queries and the result size of the query. Further on you can show the dataset and connection names. @@ -103,20 +103,20 @@ SELECT ?taskID ?datasetLabel ?connectionLabel ?noq { ?suiteID rdf:type iont:Suite . ?suiteID iprop:experiment ?expID . ?expID iprop:dataset ?dataset . - ?dataset rdfs:label ?datasetLabel + ?dataset rdfs:label ?datasetLabel . ?expID iprop:task ?taskID . - ?taskID iprop:connection ?connection. + ?taskID iprop:connection ?connection . ?connection rdfs:label ?connectionLabel . - ?taskID iprop:NoQ ?noq. + ?taskID iprop:NoQ ?noq . } ``` -This query will show a table containing for each task, the taskID, the dataset name, the connection name and the no. of queries succesfully executed. +This query will show a table containing, for each task, the taskID, the dataset name, the connection name and the number of successfully executed queries. ## SPARQL Query statistics -If you were using SPARQL queries as your benchmark queries you can add addtional further statistics of a query, such as: does the query has a FILTER. +If you were using SPARQL queries as your benchmark queries, you can add additional statistics of a query, such as, if the query has a FILTER: ```sparql PREFIX rdf: @@ -134,19 +134,19 @@ SELECT ?executedQuery ?qps ?hasFilter ?queryText { } ``` -This provides the qps value, if the SPARQL query has a filter and the actual query string. +This provides the qps value, a value that tells you, if the SPARQL query has a filter and the actual query string. ## Ontology The results' ontology (description of what each property and class means) can be -found [here](http://iguana-benchmark.eu/ontology/4.0.0/iguana.owl) +found [here](http://iguana-benchmark.eu/ontology/4.0.0/iguana.owl). -## Adding LSQ analyzation +## Adding LSQ Analyzation -If you're using SPARQL and want some more indepth analysation of the query statistics, you can use [LSQ](https://github.com/AKSW/LSQ) to do so. +If you're using SPARQL and want some more in-depth analysation of the query statistics, you can use [LSQ](https://github.com/AKSW/LSQ) to do so. Iguana will add an `owl:sameAs` link between the SPARQL queries used in your benchmark and the equivalent LSQ query links. -Hence you can run the performance measurement using Iguana and the query analyzation using LSQ independently and combine both results afterwards +Hence, you can run the performance measurement using Iguana and the query analyzation using LSQ independently and combine both results afterwards. diff --git a/docs/usage/stresstest.md b/docs/usage/stresstest.md index 07aa31dfd..972123575 100644 --- a/docs/usage/stresstest.md +++ b/docs/usage/stresstest.md @@ -2,19 +2,20 @@ Iguanas implemented Stresstest benchmark task tries to emulate a real case scenario under which an endpoint or application is under high stress. -As in real life endpoints might get multiple simultaneous request within seconds, it is very important to verify that -you application can handle this. +As in real life, endpoints might get multiple simultaneous requests within seconds, thus it is very important to verify that +your application can handle that. + +The stresstest emulates users or applications which will flood the endpoint using a set of queries for a specific +amount of time or a specific amount of executed queries. +Each simulated user is called a worker in the following. -The stresstest emulates users or applications which will bombard the endpoint using a set of queries for a specific -amount of time or a specific amount of queries executed. -Each simulated user is called Worker in the following. As you might want to test read and write performance or just want to emulate different user behaviour, the stresstest -allows to configure several workers. -Every worker configuration can additionally be started several times, hence if you want one configuration executed -multiple times, you can simply tell Iguana to run this worker configuration the specified amount of time. -However, to assure that the endpoint can't just cache the response of the first request of a query, every worker starts -at a pre-determined random query, meaning that the single worker will always start at that query to assure fairness in -benchmark comparisons, while every worker will start at a different query. +allows you to configure several workers. + +Every worker configuration can additionally be started as several simultaneous instances if you want one configuration to be executed +multiple times. +However, to assure that the endpoint can't just cache the response of the first request of a query, every worker will start +at a pre-determined random query. ## Configuration @@ -25,7 +26,7 @@ tasks: - className: "Stresstest" ``` -Further on you need to configure the Stresstest using the configuration parameter like: +Further on you have to configure the Stresstest with the configuration parameter: ```yaml tasks: @@ -35,58 +36,58 @@ tasks: ... ``` -As an end restriction you can either use `timeLimit` which will stop the stresstest after the specified amount in ms or -you can set `noOfQueryMixes` which stops every worker after they executed the amount of queries in the provided query -set. +As an end restriction, you can either use `timeLimit` which will stop the stresstest after the specified amount of time in milliseconds, or +you can set `noOfQueryMixes` which stops every worker after they have executed the specified amount of times every query in the specified location. -Additionally, to either `timeLimit` or `noOfQueryMixes` you can set the following parameters +Additionally, you have to set up these parameters: * workers * warmup (optional) ### Workers (simulated Users) -Further on you have to add which workers to use. -As described above you can set different worker configurations. +As previously mentioned, you have to set up workers for your stresstest by providing +a configuration for each worker. Let's look at an example: ```yaml - className: "Stresstest" - timeLimit: 600000 - workers: - - threads: 4 - className: "HttpGetWorker" - queries: - location: "/path/to/your/queries.txt" - - threads: 16 - className: "HttpGetWorker" - queries: - location: "/other/queries.txt" - fixedLatency: 5000 + configuration: + timeLimit: 600000 + workers: + - threads: 4 + className: "HttpGetWorker" + queries: + location: "/path/to/your/queries.txt" + - threads: 16 + className: "HttpGetWorker" + queries: + location: "/other/queries.txt" + fixedLatency: 5000 ``` -In this example we have two different worker configurations we want to use. The first want will create 4 `HttpGetWorker` -s using queries at `/path/to/your/queries.txt` without any latency thus every query will be executed immediately after -another. -The second worker configuration will execute 16 `HttpGetWorker`s using queries at `/other/queries.txt` using a fixed -waiting time of `5000ms` between each query. -Hence, every worker will execute their queries independently of each other but will wait 5s after each of their query -execution before executing the next one. -This configuration may simulate that we have a few Users requesting your endpoint locally (e.g. some of your application +In this example, we have two different worker configurations we want to use. + +The first one will create 4 workers of the class `HttpGetWorker`, that use queries located at `/path/to/your/queries.txt`. Each worker will execute their queries without any added latency between them. + +The second worker configuration will create 16 workers of the class `HttpGetWorker`, that use queries located at `/other/queries.txt`. Each worker will execute their queries using a fixed waiting time of `5000ms` between each query. +In detail, every worker will execute their queries independently of each other, but each will wait for 5s after executing one of their own queries before executing the next one. + +This configuration may simulate that we have a few users requesting your endpoint locally (e.g. some of your application relying on your database) and several users querying your endpoint from outside the network where we would have network -latency and other interferences which we will try to simulate with 5s. +latencies and other interferences. We try to simulate this behaviour with the added latency of five seconds in between queries. -A full list of supported workers and their parameters can be found at [Supported Workers](../workers) +A full list of supported workers and their parameters can be found at [Supported Workers](../workers). -In this example our Stresstest would create 20 workers, which will simultaneously request the endpoint for 60000ms (10 +In this example our Stresstest would create in total 20 workers, which will simultaneously request the endpoint for 600000ms (10 minutes). #### Query Handling -The `queries` parameter lets the worker know what queries will be used. +The `queries` parameter lets the worker know what queries should be used. The default is to have a single text file with one query per line (could be SQL, SPARQL, a whole RDF document). -You can set the query handling like the following: +The query handling may be set up like the following: ```yaml workers: @@ -100,29 +101,29 @@ workers: ... ``` -To see further configurations of the query handling see [Supported Queries](../queries/) +To see further configurations of the query handling see [Supported Queries](./queries) ### Warmup -Additionaly to these you can optionally set a warmup, which will aim to let the system be benchmarked under a normal -situation (Some times a database is faster when it was already running for a bit) -The configuration is similar to the stresstest itself you can set a `timeLimit` (however not a certain no of query -executions), you can set different `workers` to use. -Each worker needs its own `queries` configuration. +Additionally, you can optionally set up a warmup for your stresstest, which aims to let the system get benchmarked under a normal +situation (sometimes a database is faster when it was already running for a while). + +The configuration is similar to the stresstest itself. You can set a `timeLimit` (however, you can not specify a `noOfQueryMixes`), and you can use different `workers`. -You can set the Warmup as following: +You can set the Warmup as follows: ```yaml tasks: - className: "Stresstest" - warmup: - timeLimit: 600000 - workers: + configuration: + warmup: + timeLimit: 600000 + workers: ... ``` -That's it. -A full example might look like this +## Example +A full example of the stresstest configuration may look like this: ```yaml tasks: @@ -134,8 +135,7 @@ tasks: warmup: # 10 minutes (is in ms) timeLimit: 600000 - # queryHandler could be set too, same as in the stresstest configuration, otherwise the same queryHandler will be use. - # workers are set the same way as in the configuration part + # workers in the warmup are set the same way as in the main configuration workers: - threads: 1 className: "HttpGetWorker" @@ -157,5 +157,5 @@ tasks: ## References -* [Supported Queries](../queries/) +* [Supported Queries](../queries) * [Supported Workers](../workers) diff --git a/docs/usage/tutorial.md b/docs/usage/tutorial.md index 34ac58b4d..9380acbf1 100644 --- a/docs/usage/tutorial.md +++ b/docs/usage/tutorial.md @@ -74,6 +74,7 @@ Then, insert this basic configuration, which should suffice for this tutorial, i com.bigdata.rdf.store.AbstractTripleStore.statementIdentifiers=true com.bigdata.journal.AbstractJournal.bufferMode=DiskRW com.bigdata.journal.AbstractJournal.file=blazegraph.jnl +com.bigdata.rdf.store.AbstractTripleStore.quads=false ``` Now we can go ahead and create our script files. First create the files with: @@ -92,7 +93,7 @@ blazegraph and load the given datasets: cd ../blazegraph # load the dataset file, which will be set as the first script argument -java -cp blazegraph.jar com.bigdata.rdf.store.DataLoader -defaultGraph http://example.org p.properties $1 +java -cp blazegraph.jar com.bigdata.rdf.store.DataLoader p.properties $1 # start blazegraph with 4 GB ram java -Xmx4g -server -jar blazegraph.jar & @@ -181,7 +182,7 @@ Create a file called `benchmark-suite.yml`: touch benchmark-suite.yml ``` -Add the following subsections to this file, or simply go to [#Full Configuration](configuration#full-example) and add +Add the following subsections to this file, or simply go to the [Full Configuration](#full-configuration) and add the whole piece to it. Be aware that Iguana will be started from the directory `myBenchmark/iguana/`, thus paths will need to use `../` to get the correct paths. @@ -224,6 +225,7 @@ This will execute the appropriate script with the current dataset as the argumen For example, the pre-task script execution for fuseki and the ekaw dataset will look like this: + ```bash ./fuseki/load-and-start.sh ../datasets/ekaw-2012-complete-alignments.rdf ``` @@ -344,9 +346,9 @@ SELECT ?taskID ?datasetLabel ?connectionLabel ?noq { ?expID iprop:dataset ?dataset . ?dataset rdfs:label ?datasetLabel . ?expID iprop:task ?taskID . - ?taskID iprop:connection ?connection. + ?taskID iprop:connection ?connection . ?connection rdfs:label ?connectionLabel . - ?taskID iprop:NoQ ?noq. + ?taskID iprop:NoQ ?noq . } ``` diff --git a/docs/usage/workers.md b/docs/usage/workers.md index e1cec29ab..63a4c60e0 100644 --- a/docs/usage/workers.md +++ b/docs/usage/workers.md @@ -5,83 +5,83 @@ requesting your system until it should stop. In a task (e.g. the [stresstest](../stresstest/)) you can configure several worker configurations which will then be used inside the task. -Every worker configuration can additionally be started several times, hence if you want one configuration executed -multiple times, you can simply tell Iguana to run this worker configuration the specified amount of time. +Every worker configuration can additionally be started several times, if you want a configuration to be executed +multiple times. However, to assure that the endpoint can't just cache the response of the first request of a query, every worker starts at a pre-determined random query, meaning that the single worker will always start at that query to assure fairness in benchmark comparisons, while every worker will start at a different query. There are a few workers implemented, which can be seperated into two main categories -* Http Workers +* HTTP Workers * CLI Workers ## Common Configuration Every worker has the following configuration parameters: -| parameter | optional | default | description | -|-----------------|----------|--------------------------|--------------------------------------------------------------------------------------------------------------------------------| -| threads | no | | The amount of workers to start using this worker configuration | -| queries | no | | Configuration for the queries this worker should use. (see [SupportedQueries](../queries)) | -| timeOut | yes | 180000 (3 minutes) | The timeout in MS after a query should be aborted | -| fixedLatency | yes | 0 | If the value (in MS) should be waited between each query. Simulating network latency or user behaviour. | -| gaussianLatency | yes | 0 | A random value between `[0, 2*value]` (in MS) will be waited between each query. Simulating network latency or user behaviour. | +| parameter | optional | default | description | +|-----------------|----------|--------------------------|------------------------------------------------------------------------------------------------------------------------------------------| +| threads | no | | The amount of workers to start using this worker configuration | +| queries | no | | Configuration for the queries this worker should use. (see [Supported Queries](../queries)) | +| timeOut | yes | 180000 (3 minutes) | The timeout in milliseconds after a query should be aborted | +| fixedLatency | yes | 0 | The amount of time (in milliseconds) a worker should wait after each query. It's used to simulate network latency or user behaviour. | +| gaussianLatency | yes | 0 | A random value between `[0, 2*value]` (in milliseconds) will be waited between each query. Simulating network latency or user behaviour. | -## Http Workers +## HTTP Workers -These Workers can be used to benchmark Http Applications (such as a SPARQL endpoint). +These Workers can be used to benchmark HTTP applications (such as SPARQL endpoints). -### Http Get Worker +### HTTP GET Worker -A Http worker using GET requests. +An HTTP worker using GET requests. This worker will use the `endpoint` of the connection. This worker has several configurations listed in the following table: -| parameter | optional | default | description | -|-----------------|----------|--------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------| -| parameterName | yes | query | The GET parameter to set the query as value to. (see also [Supported Queries](../queries) ) | -| responseType | yes | | The content type the endpoint should return. Setting the `Accept: ` header | +| parameter | optional | default | description | +|-----------------|----------|--------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------| +| parameterName | yes | query | The GET parameter to set the query as value to. (see also [Supported Queries](./queries)) | +| responseType | yes | | The content type the endpoint should return. Setting the `Accept: ` header | Let's look at an example: ```yaml ... workers: - - threads: 1 - className: "HttpGetWorker" - queries: - ... - timeOut: 180000 - parameterName: "text" + - threads: 1 + className: "HttpGetWorker" + queries: + ... + timeOut: 180000 + parameterName: "text" ``` This will use one HttpGetWorker using a timeout of 3 minutes and the get parameter `text` to request the query through. -### Http Post Worker +### HTTP POST Worker -A Http worker using POST requests. +An HTTP worker using POST requests. This worker will use the `updateEndpoint` of the connection. This worker has several configurations listed in the following table: -| parameter | optional | default | description | -|-----------------|----------|--------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------| -| parameterName | yes | query | the GET parameter to set the query as value to. (see also [Supported Queries](../queries) ) | -| contentType | yes | `text/plain` | The content type of the update queries. Setting the `Content-Type: ` header | -| responseType | yes | | The content type the endpoint should return. Setting the `Accept: ` header | +| parameter | optional | default | description | +|-----------------|----------|--------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------| +| parameterName | yes | query | the GET parameter to set the query as value to. (see also [Supported Queries](../queries)) | +| contentType | yes | `text/plain` | The content type of the update queries. Setting the `Content-Type: ` header | +| responseType | yes | | The content type the endpoint should return. Setting the `Accept: ` header | Let's look at an example: ```yaml ... workers: - - threads: 1 - className: "HttpPostWorker" - queries: - ... - timeOut: 180000 + - threads: 1 + className: "HttpPostWorker" + queries: + ... + timeOut: 180000 ``` This will use one HttpPostWorker using a timeout of 3 minutes. @@ -90,7 +90,7 @@ This will use one HttpPostWorker using a timeout of 3 minutes. Through the update of the query handling the `SPARQLWorker` is no longer different to the `HttpGetWorker`, since the language parameter is set in the `queries` config rather than in the worker config. -Therefore we removed the SPARQL worker. +Therefore, we removed the SPARQL worker. ### SPARQL UPDATE Worker @@ -116,12 +116,12 @@ An Example: ```yaml ... workers: - - threads: 1 - className: "UPDATEWorker" - queries: - ... - timeOut: 180000 - timerStrategy: "FIXED" + - threads: 1 + className: "UPDATEWorker" + queries: + ... + timeOut: 180000 + timerStrategy: "FIXED" ``` ## CLI Workers diff --git a/docs/usage/workflow.md b/docs/usage/workflow.md index 6ec5d42fc..cded5d673 100644 --- a/docs/usage/workflow.md +++ b/docs/usage/workflow.md @@ -1,6 +1,7 @@ # Workflow -Iguana will first parse configuration and afterwards will execute each task for each connection for each dataset. +Iguana will first parse the configuration file. +Afterwards it will execute each task for each connection for each dataset. Imagine it like the following: @@ -12,4 +13,3 @@ Imagine it like the following: 3. collect and calculate results 4. write results 5. execute post script hook - From 89507762a7d50c5f29503fea092909d46c8393ee Mon Sep 17 00:00:00 2001 From: nck-mlcnv <32801560+nck-mlcnv@users.noreply.github.com> Date: Fri, 31 Mar 2023 15:45:13 +0200 Subject: [PATCH 37/47] ignore cli tests --- .../java/org/aksw/iguana/cc/utils/CLIProcessManagerTest.java | 2 ++ .../java/org/aksw/iguana/cc/worker/impl/CLIWorkersTests.java | 2 ++ 2 files changed, 4 insertions(+) diff --git a/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/utils/CLIProcessManagerTest.java b/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/utils/CLIProcessManagerTest.java index e6a232935..f16cf6319 100644 --- a/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/utils/CLIProcessManagerTest.java +++ b/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/utils/CLIProcessManagerTest.java @@ -1,11 +1,13 @@ package org.aksw.iguana.cc.utils; +import org.junit.Ignore; import org.junit.Test; import java.io.IOException; import static org.junit.Assert.*; +@Ignore("CLI doesn't work right now") public class CLIProcessManagerTest { @Test diff --git a/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/worker/impl/CLIWorkersTests.java b/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/worker/impl/CLIWorkersTests.java index faf5d03a4..8ec21bb3d 100644 --- a/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/worker/impl/CLIWorkersTests.java +++ b/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/worker/impl/CLIWorkersTests.java @@ -5,6 +5,7 @@ import org.aksw.iguana.commons.constants.COMMON; import org.junit.After; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; import java.io.File; @@ -14,6 +15,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +@Ignore("CLI workers don't work right now") public class CLIWorkersTests { private File f; From f7db1894292cc619839cdcdacb51c1adeef8856f Mon Sep 17 00:00:00 2001 From: nck-mlcnv <32801560+nck-mlcnv@users.noreply.github.com> Date: Fri, 31 Mar 2023 15:46:45 +0200 Subject: [PATCH 38/47] update ci badge --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 52a9ab3ab..6401ebafe 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # IGUANA -![Java CI with Maven](https://github.com/dice-group/IGUANA/workflows/Java%20CI%20with%20Maven/badge.svg) +[![ci](https://github.com/dice-group/IGUANA/actions/workflows/ci.yml/badge.svg)](https://github.com/dice-group/IGUANA/actions/workflows/ci.yml)

IGUANA Logo From 06ba515b23e21ef9e89c2b1698212a04a24e05e9 Mon Sep 17 00:00:00 2001 From: nck-mlcnv <32801560+nck-mlcnv@users.noreply.github.com> Date: Wed, 5 Apr 2023 16:27:18 +0200 Subject: [PATCH 39/47] remove parenthesis --- .../main/java/org/aksw/iguana/cc/config/IguanaConfig.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/config/IguanaConfig.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/config/IguanaConfig.java index 222f22dca..221776c55 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/config/IguanaConfig.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/config/IguanaConfig.java @@ -50,13 +50,13 @@ public class IguanaConfig { private List connections; @JsonProperty(required = true) private List tasks; - @JsonProperty() + @JsonProperty private String preScriptHook; - @JsonProperty() + @JsonProperty private String postScriptHook; - @JsonProperty() + @JsonProperty private List metrics; - @JsonProperty() + @JsonProperty private List storages; From 5f6f6ce461587fc3d761b9bfe20ddc385b6597da Mon Sep 17 00:00:00 2001 From: nck-mlcnv <32801560+nck-mlcnv@users.noreply.github.com> Date: Wed, 5 Apr 2023 16:30:34 +0200 Subject: [PATCH 40/47] fix spelling in javadocs --- .../aksw/iguana/cc/query/pattern/PatternHandler.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/pattern/PatternHandler.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/pattern/PatternHandler.java index 5194ddb6b..63cb1a907 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/pattern/PatternHandler.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/pattern/PatternHandler.java @@ -13,9 +13,9 @@ import java.util.regex.Pattern; /** - * This class is used to instantiate SPARQL pattern queries. - * It will replace create a SPARQL query to request values, for the variables - * in the pattern query, from a provided SPARQL endpoint. + * This class is used to instantiate SPARQL pattern queries.
+ * It will create and execute a SPARQL query against the provided SPARQL endpoint, that will retrieve fitting values for + * the variables in the pattern query. *

* The instantiated queries are located in a text file, which is created at the given location. * If a fitting query file is already present, the queries will not be instantiated again. @@ -39,8 +39,8 @@ public PatternHandler(Map config, QuerySource querySource) { } /** - * This method will generate the queries from the given patterns, writes them - * to a file, and returns a QuerySource based on that file. + * This method will generate the queries from the given patterns, write them + * to a file, and return a QuerySource based on that file. * The QuerySource is then used in the QueryHandler to get the queries. * * @return QuerySource containing the instantiated queries From b28283d66c1a9024fa650943c4ec9b3069ab3000 Mon Sep 17 00:00:00 2001 From: nck-mlcnv <32801560+nck-mlcnv@users.noreply.github.com> Date: Wed, 5 Apr 2023 16:30:48 +0200 Subject: [PATCH 41/47] change .gitignore --- .gitignore | 63 ++++++++++++++++-------------------------------------- 1 file changed, 18 insertions(+), 45 deletions(-) diff --git a/.gitignore b/.gitignore index b998c1d33..bc02395e7 100644 --- a/.gitignore +++ b/.gitignore @@ -5,8 +5,8 @@ tmp_ser **/queryInstances/* -# Created by https://www.toptal.com/developers/gitignore/api/java,maven,intellij,eclipse -# Edit at https://www.toptal.com/developers/gitignore?templates=java,maven,intellij,eclipse +# Created by https://www.toptal.com/developers/gitignore/api/java,maven,intellij+all,eclipse +# Edit at https://www.toptal.com/developers/gitignore?templates=java,maven,intellij+all,eclipse ### Eclipse ### .metadata @@ -74,7 +74,7 @@ local.properties # Spring Boot Tooling .sts4-cache/ -### Intellij ### +### Intellij+all ### # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 @@ -108,14 +108,14 @@ local.properties # When using Gradle or Maven with auto-import, you should exclude module files, # since they will be recreated, and may cause churn. Uncomment if using # auto-import. -.idea/artifacts -.idea/compiler.xml -.idea/jarRepositories.xml -.idea/modules.xml -.idea/*.iml -.idea/modules -*.iml -*.ipr +# .idea/artifacts +# .idea/compiler.xml +# .idea/jarRepositories.xml +# .idea/modules.xml +# .idea/*.iml +# .idea/modules +# *.iml +# *.ipr # CMake cmake-build-*/ @@ -153,39 +153,14 @@ fabric.properties # Android studio 3.1+ serialized cache file .idea/caches/build_file_checksums.ser -### Intellij Patch ### -# Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721 - -# *.iml -# modules.xml -# .idea/misc.xml -# *.ipr - -# Sonarlint plugin -# https://plugins.jetbrains.com/plugin/7973-sonarlint -.idea/**/sonarlint/ +### Intellij+all Patch ### +# Ignore everything but code style settings and run configurations +# that are supposed to be shared within teams. -# SonarQube Plugin -# https://plugins.jetbrains.com/plugin/7238-sonarqube-community-plugin -.idea/**/sonarIssues.xml +.idea/* -# Markdown Navigator plugin -# https://plugins.jetbrains.com/plugin/7896-markdown-navigator-enhanced -.idea/**/markdown-navigator.xml -.idea/**/markdown-navigator-enh.xml -.idea/**/markdown-navigator/ - -# Cache file creation bug -# See https://youtrack.jetbrains.com/issue/JBR-2257 -.idea/$CACHE_FILE$ - -# CodeStream plugin -# https://plugins.jetbrains.com/plugin/12206-codestream -.idea/codestream.xml - -# Azure Toolkit for IntelliJ plugin -# https://plugins.jetbrains.com/plugin/8053-azure-toolkit-for-intellij -.idea/**/azureSettings.xml +!.idea/codeStyles +!.idea/runConfigurations ### Java ### # Compiled class file @@ -232,6 +207,4 @@ buildNumber.properties # JDT-specific (Eclipse Java Development Tools) .classpath -# End of https://www.toptal.com/developers/gitignore/api/java,maven,intellij,eclipse - - +# End of https://www.toptal.com/developers/gitignore/api/java,maven,intellij+all,eclipse From 6b07d0408ae71f3cd7bc3e89a9b6a6396d08e6fd Mon Sep 17 00:00:00 2001 From: nck-mlcnv <32801560+nck-mlcnv@users.noreply.github.com> Date: Wed, 5 Apr 2023 16:44:40 +0200 Subject: [PATCH 42/47] remove SPARQLWorker --- .../iguana/cc/worker/impl/SPARQLWorker.java | 22 ------------------- .../iguana/cc/worker/UPDATEWorkerTest.java | 3 +-- 2 files changed, 1 insertion(+), 24 deletions(-) delete mode 100644 iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/impl/SPARQLWorker.java diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/impl/SPARQLWorker.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/impl/SPARQLWorker.java deleted file mode 100644 index a72af5da0..000000000 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/worker/impl/SPARQLWorker.java +++ /dev/null @@ -1,22 +0,0 @@ -package org.aksw.iguana.cc.worker.impl; - -import org.aksw.iguana.cc.config.elements.Connection; -import org.aksw.iguana.commons.annotation.Nullable; -import org.aksw.iguana.commons.annotation.Shorthand; - -import java.util.Map; - - -/** - * A Worker using SPARQL 1.1 to create service request. - * - * @author f.conrads - */ -@Shorthand("SPARQLWorker") -public class SPARQLWorker extends HttpGetWorker { - - public SPARQLWorker(String taskID, Integer workerID, Connection connection, Map queries, @Nullable Integer timeLimit, @Nullable Integer timeOut, @Nullable Integer fixedLatency, @Nullable Integer gaussianLatency, @Nullable String parameterName, @Nullable String responseType) { - super(taskID, workerID, connection, queries, timeLimit, timeOut, fixedLatency, gaussianLatency, parameterName, responseType); - } - -} diff --git a/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/worker/UPDATEWorkerTest.java b/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/worker/UPDATEWorkerTest.java index 3a0258bbe..53fe806c9 100644 --- a/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/worker/UPDATEWorkerTest.java +++ b/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/worker/UPDATEWorkerTest.java @@ -1,7 +1,6 @@ package org.aksw.iguana.cc.worker; import org.aksw.iguana.cc.config.elements.Connection; -import org.aksw.iguana.cc.worker.impl.SPARQLWorker; import org.aksw.iguana.cc.worker.impl.UPDATEWorker; import org.aksw.iguana.cc.worker.impl.update.UpdateTimer; import org.aksw.iguana.commons.time.TimeUtils; @@ -43,7 +42,7 @@ public UPDATEWorkerTest(String timerStrategy, Map queriesFile, i //warmup Map warmupQueries = new HashMap<>(); warmupQueries.put("location", "src/test/resources/workers/single-query.txt"); - SPARQLWorker worker = new SPARQLWorker("", 1, getConnection(), warmupQueries, null, null, null, null, null, null); + UPDATEWorker worker = new UPDATEWorker("", 1, getConnection(), warmupQueries, null, null, null, null, null); worker.executeQuery("INSERT DATA {", "1"); fastServerContainer.getTimes().clear(); fastServerContainer.getEncodedAuth().clear(); From 99d40deb812a0baee67094c358aea4ad9586df7d Mon Sep 17 00:00:00 2001 From: nck-mlcnv <32801560+nck-mlcnv@users.noreply.github.com> Date: Wed, 5 Apr 2023 17:05:10 +0200 Subject: [PATCH 43/47] fix javadocs and rename the method "initPattern" --- .../iguana/cc/query/handler/QueryHandler.java | 28 +++++++++++-------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/handler/QueryHandler.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/handler/QueryHandler.java index d7b653988..7cfed6a56 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/handler/QueryHandler.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/handler/QueryHandler.java @@ -25,7 +25,7 @@ import java.util.Map; /** - * This QueryHandler which is used by every worker who is extending the AbstractWorker. + * The QueryHandler is used by every worker that extends the AbstractWorker. * It initializes the QuerySource, QuerySelector, QuerySet and, if needed, PatternHandler. * After the initialization, it provides the next query to the worker using the generated QuerySource * and the order given by the QuerySelector. @@ -55,11 +55,7 @@ public QueryHandler(Map config, Integer workerID) { this.location = (String) config.get("location"); - if (config.containsKey("pattern")) { - initPattern(); - } else { - initQuerySet(); - } + initQuerySet(); initQuerySelector(); initLanguageProcessor(); @@ -99,11 +95,12 @@ public LanguageProcessor getLanguageProcessor() { } /** - * Will initialize the PatternHandler if a pattern config is given. - * The PatternHandler uses the original query source and generates a new query source - * which is used during the benchmark execution. + * This method initializes the PatternHandler if a pattern config is given, therefore + * this.config.get("pattern") should return an appropriate pattern configuration and not + * null. The PatternHandler uses the original query source to generate a new query source and set with + * the instantiated queries. */ - private void initPattern() { + private void initPatternQuerySet() { Map patternConfig = (Map) this.config.get("pattern"); PatternHandler patternHandler = new PatternHandler(patternConfig, createQuerySource()); @@ -126,8 +123,17 @@ private void initQuerySet(QuerySource querySource) { } } + /** + * This method initializes the QuerySet for the QueryHandler. If a pattern configuration is specified, this method + * will execute initPatternQuerySet to create the QuerySet. + */ private void initQuerySet() { - initQuerySet(createQuerySource()); + if(this.config.containsKey("pattern")) { + initPatternQuerySet(); + } + else { + initQuerySet(createQuerySource()); + } } /** From 3228e0c021774b503c56bb8047b42834cf6f6f1f Mon Sep 17 00:00:00 2001 From: nck-mlcnv <32801560+nck-mlcnv@users.noreply.github.com> Date: Wed, 5 Apr 2023 18:21:17 +0200 Subject: [PATCH 44/47] remove unnecessary abstraction --- .../iguana/cc/query/handler/QueryHandler.java | 10 ++--- .../cc/query/pattern/PatternHandler.java | 8 ++-- .../query/selector/AbstractQuerySelector.java | 15 ------- .../cc/query/selector/QuerySelector.java | 3 +- .../selector/impl/LinearQuerySelector.java | 13 +++--- .../selector/impl/RandomQuerySelector.java | 8 ++-- .../iguana/cc/query/set/AbstractQuerySet.java | 41 +++++++++++++---- .../aksw/iguana/cc/query/set/QuerySet.java | 40 ----------------- .../cc/query/set/impl/FileBasedQuerySet.java | 4 +- .../cc/query/set/impl/InMemQuerySet.java | 4 +- .../cc/query/source/AbstractQuerySource.java | 34 ++++++++++++-- .../iguana/cc/query/source/QuerySource.java | 45 ------------------- .../pattern/PatternBasedQueryHandlerTest.java | 10 ++--- .../cc/query/pattern/PatternHandlerTest.java | 4 +- 14 files changed, 99 insertions(+), 140 deletions(-) delete mode 100644 iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/selector/AbstractQuerySelector.java delete mode 100644 iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/QuerySet.java delete mode 100644 iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/QuerySource.java diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/handler/QueryHandler.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/handler/QueryHandler.java index 7cfed6a56..751e533bc 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/handler/QueryHandler.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/handler/QueryHandler.java @@ -6,10 +6,10 @@ import org.aksw.iguana.cc.query.selector.QuerySelector; import org.aksw.iguana.cc.query.selector.impl.LinearQuerySelector; import org.aksw.iguana.cc.query.selector.impl.RandomQuerySelector; -import org.aksw.iguana.cc.query.set.QuerySet; +import org.aksw.iguana.cc.query.set.AbstractQuerySet; import org.aksw.iguana.cc.query.set.impl.FileBasedQuerySet; import org.aksw.iguana.cc.query.set.impl.InMemQuerySet; -import org.aksw.iguana.cc.query.source.QuerySource; +import org.aksw.iguana.cc.query.source.AbstractQuerySource; import org.aksw.iguana.cc.query.source.impl.FileLineQuerySource; import org.aksw.iguana.cc.query.source.impl.FileSeparatorQuerySource; import org.aksw.iguana.cc.query.source.impl.FolderQuerySource; @@ -45,7 +45,7 @@ public class QueryHandler { protected QuerySelector querySelector; - protected QuerySet querySet; + protected AbstractQuerySet querySet; protected LanguageProcessor langProcessor; @@ -113,7 +113,7 @@ private void initPatternQuerySet() { * * @param querySource The QuerySource which contains the queries. */ - private void initQuerySet(QuerySource querySource) { + private void initQuerySet(AbstractQuerySource querySource) { this.caching = (Boolean) this.config.getOrDefault("caching", true); if (this.caching) { @@ -144,7 +144,7 @@ private void initQuerySet() { * * @return The QuerySource which contains the queries. */ - private QuerySource createQuerySource() { + private AbstractQuerySource createQuerySource() { Object formatObj = this.config.getOrDefault("format", "one-per-line"); if (formatObj instanceof Map) { Map format = (Map) formatObj; diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/pattern/PatternHandler.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/pattern/PatternHandler.java index 63cb1a907..4d8392000 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/pattern/PatternHandler.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/pattern/PatternHandler.java @@ -1,6 +1,6 @@ package org.aksw.iguana.cc.query.pattern; -import org.aksw.iguana.cc.query.source.QuerySource; +import org.aksw.iguana.cc.query.source.AbstractQuerySource; import org.aksw.iguana.cc.query.source.impl.FileLineQuerySource; import org.apache.jena.query.*; import org.slf4j.Logger; @@ -26,12 +26,12 @@ public class PatternHandler { private static final Logger LOGGER = LoggerFactory.getLogger(PatternHandler.class); private final Map config; - private final QuerySource querySource; + private final AbstractQuerySource querySource; private String endpoint; private Long limit; private String outputFolder; - public PatternHandler(Map config, QuerySource querySource) { + public PatternHandler(Map config, AbstractQuerySource querySource) { this.config = config; this.querySource = querySource; @@ -45,7 +45,7 @@ public PatternHandler(Map config, QuerySource querySource) { * * @return QuerySource containing the instantiated queries */ - public QuerySource generateQuerySource() { + public AbstractQuerySource generateQuerySource() { File cacheFile = new File(this.outputFolder + File.separator + this.querySource.hashCode()); if (cacheFile.exists()) { diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/selector/AbstractQuerySelector.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/selector/AbstractQuerySelector.java deleted file mode 100644 index 044412ff2..000000000 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/selector/AbstractQuerySelector.java +++ /dev/null @@ -1,15 +0,0 @@ -package org.aksw.iguana.cc.query.selector; - -/** - * The QuerySelector is used to get the next query index. - * - * @author frensing - */ -public abstract class AbstractQuerySelector implements QuerySelector { - - protected int size; - - public AbstractQuerySelector(int size) { - this.size = size; - } -} diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/selector/QuerySelector.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/selector/QuerySelector.java index 82fc7c66b..e66a954bc 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/selector/QuerySelector.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/selector/QuerySelector.java @@ -1,12 +1,13 @@ package org.aksw.iguana.cc.query.selector; /** - * The QuerySelector that is used to get the next query index. + * The QuerySelector provides a method to retrieve the index of a query, that should be executed next.
* It is used by the QueryHandler to get the next query. * * @author frensing */ public interface QuerySelector { + /** * This method gives the next query index that should be used. * diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/selector/impl/LinearQuerySelector.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/selector/impl/LinearQuerySelector.java index ee763b8bd..a8a5abe37 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/selector/impl/LinearQuerySelector.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/selector/impl/LinearQuerySelector.java @@ -1,20 +1,23 @@ package org.aksw.iguana.cc.query.selector.impl; -import org.aksw.iguana.cc.query.selector.AbstractQuerySelector; +import org.aksw.iguana.cc.query.selector.QuerySelector; /** - * This QuerySelector is used to get the next query index in a linear order. - * If the last query is reached it starts again at the first query. + * This QuerySelector is used to get the next query index in a linear order. If the last query is reached it starts + * again at the first query. *

* It is used by the QueryHandler to get the next query. * * @author frensing */ -public class LinearQuerySelector extends AbstractQuerySelector { +public class LinearQuerySelector implements QuerySelector { + protected int querySelector; + private int size; + public LinearQuerySelector(int size) { - super(size); + this.size = size; } @Override diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/selector/impl/RandomQuerySelector.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/selector/impl/RandomQuerySelector.java index 6cd1e4a3e..fdff4a248 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/selector/impl/RandomQuerySelector.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/selector/impl/RandomQuerySelector.java @@ -1,6 +1,6 @@ package org.aksw.iguana.cc.query.selector.impl; -import org.aksw.iguana.cc.query.selector.AbstractQuerySelector; +import org.aksw.iguana.cc.query.selector.QuerySelector; import java.util.Random; @@ -11,12 +11,14 @@ * * @author frensing */ -public class RandomQuerySelector extends AbstractQuerySelector { +public class RandomQuerySelector implements QuerySelector { protected Random querySelector; + private int size; + public RandomQuerySelector(int size, long seed) { - super(size); + this.size = size; this.querySelector = new Random(seed); } diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/AbstractQuerySet.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/AbstractQuerySet.java index df55eada2..a22690ef8 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/AbstractQuerySet.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/AbstractQuerySet.java @@ -1,35 +1,60 @@ package org.aksw.iguana.cc.query.set; -import org.aksw.iguana.cc.query.source.QuerySource; +import org.aksw.iguana.cc.query.source.AbstractQuerySource; + +import java.io.IOException; /** - * The abstract class for a QuerySet. - * It implements the basic functions that are shared for all QuerySets. + * The abstract class for a QuerySet. A query set provides the queries to the QueryHandler. * * @author frensing */ -public abstract class AbstractQuerySet implements QuerySet { - protected QuerySource querySource; +public abstract class AbstractQuerySet { + + /** This is the QuerySource from which the queries should be retrieved. */ + protected AbstractQuerySource querySource; + /** A name for the query set. This is a part of the queryIDs. */ protected String name; - public AbstractQuerySet(String name, QuerySource querySource) { + public AbstractQuerySet(String name, AbstractQuerySource querySource) { this.name = name; this.querySource = querySource; } - @Override + /** + * This method returns the amount of queries in the query set. + * + * @return The amount of queries in the query set + */ public int size() { return this.querySource.size(); } - @Override + /** + * This method returns the name of the query set. + * + * @return The name of the query set + */ public String getName() { return this.name; } + /** + * This method returns the hashcode of the query set which is the hashcode of the query source. + * + * @return The hashcode of the query set + */ @Override public int hashCode() { return this.querySource.hashCode(); } + + /** + * This method returns a query at the given position. + * + * @param pos Position of the query in the set + * @return The query at position pos + */ + public abstract String getQueryAtPos(int pos) throws IOException; } diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/QuerySet.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/QuerySet.java deleted file mode 100644 index 213c85d59..000000000 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/QuerySet.java +++ /dev/null @@ -1,40 +0,0 @@ -package org.aksw.iguana.cc.query.set; - - -import java.io.IOException; - -/** - * A query set provides the Queries to the QueryHandler. - */ -public interface QuerySet { - - /** - * Gets a query at the position pos. - * - * @param pos Position of the query in the set - * @return The query at position pos - */ - String getQueryAtPos(int pos) throws IOException; - - /** - * Gets no. of queries in the query set - * - * @return The no of queries in the query set - */ - int size(); - - /** - * Gets the name of the query set - * - * @return The name of the query set - */ - String getName(); - - /** - * Gets the hashcode of the query set which is the hashcode of the query source - * - * @return The hashcode of the query set - */ - @Override - int hashCode(); -} diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/impl/FileBasedQuerySet.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/impl/FileBasedQuerySet.java index ecba5aa13..19a658927 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/impl/FileBasedQuerySet.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/impl/FileBasedQuerySet.java @@ -1,7 +1,7 @@ package org.aksw.iguana.cc.query.set.impl; import org.aksw.iguana.cc.query.set.AbstractQuerySet; -import org.aksw.iguana.cc.query.source.QuerySource; +import org.aksw.iguana.cc.query.source.AbstractQuerySource; import java.io.IOException; @@ -12,7 +12,7 @@ */ public class FileBasedQuerySet extends AbstractQuerySet { - public FileBasedQuerySet(String name, QuerySource querySource) { + public FileBasedQuerySet(String name, AbstractQuerySource querySource) { super(name, querySource); } diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/impl/InMemQuerySet.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/impl/InMemQuerySet.java index db22d57cc..0ff0a0177 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/impl/InMemQuerySet.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/impl/InMemQuerySet.java @@ -1,7 +1,7 @@ package org.aksw.iguana.cc.query.set.impl; import org.aksw.iguana.cc.query.set.AbstractQuerySet; -import org.aksw.iguana.cc.query.source.QuerySource; +import org.aksw.iguana.cc.query.source.AbstractQuerySource; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -20,7 +20,7 @@ public class InMemQuerySet extends AbstractQuerySet { private List queries; - public InMemQuerySet(String name, QuerySource querySource) { + public InMemQuerySet(String name, AbstractQuerySource querySource) { super(name, querySource); loadQueries(); } diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/AbstractQuerySource.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/AbstractQuerySource.java index 2c15b0f6a..4c107c83b 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/AbstractQuerySource.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/AbstractQuerySource.java @@ -2,20 +2,48 @@ import org.aksw.iguana.cc.utils.FileUtils; +import java.io.IOException; +import java.util.List; + /** - * The abstract class for a QuerySource. - * It implements the basic functions that are shared for all QuerySources. + * The abstract class for a QuerySource.
+ * The QuerySource provides the queries to the QuerySet. It abstracts the actual format of the query files. * * @author frensing */ -public abstract class AbstractQuerySource implements QuerySource { +public abstract class AbstractQuerySource { + /** This string represents the path of the file or folder, that contains the queries. */ protected String path; public AbstractQuerySource(String path) { this.path = path; } + /** + * This method returns the amount of queries in the source. + * + * @return the number of queries in the source + */ + public abstract int size(); + + /** + * This method returns the query at the given index. + * + * @param index the index of the query counted from the first query (in the first file) + * @return String of the query + * @throws IOException + */ + public abstract String getQuery(int index) throws IOException; + + /** + * This method returns all queries in the source as a list of Strings. + * + * @return List of Strings of all queries + * @throws IOException + */ + public abstract List getAllQueries() throws IOException; + @Override public int hashCode() { return FileUtils.getHashcodeFromFileContent(this.path); diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/QuerySource.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/QuerySource.java deleted file mode 100644 index 2c0a5b210..000000000 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/QuerySource.java +++ /dev/null @@ -1,45 +0,0 @@ -package org.aksw.iguana.cc.query.source; - -import java.io.IOException; -import java.util.List; - -/** - * The QuerySource is providing the queries to the QuerySet. - * It abstracts the actual format of the query files. - * - * @author frensing - */ -public interface QuerySource { - - /** - * This method returns the number of queries in the source. - * - * @return the number of queries in the source - */ - int size(); - - /** - * This method returns the query at the given index. - * - * @param index the index of the query counted from the first query (in the first file) - * @return String of the query - * @throws IOException - */ - String getQuery(int index) throws IOException; - - /** - * This method returns all queries in the source as Strings. - * - * @return List of Strings of all queries - * @throws IOException - */ - List getAllQueries() throws IOException; - - /** - * This method returns the hashcode of the source, calculated from the file contents rather than the Java object. - * - * @return the hashcode of the source - */ - @Override - int hashCode(); -} diff --git a/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/pattern/PatternBasedQueryHandlerTest.java b/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/pattern/PatternBasedQueryHandlerTest.java index 2c082531c..92a43d63c 100644 --- a/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/pattern/PatternBasedQueryHandlerTest.java +++ b/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/pattern/PatternBasedQueryHandlerTest.java @@ -1,6 +1,6 @@ package org.aksw.iguana.cc.query.pattern; -import org.aksw.iguana.cc.query.source.QuerySource; +import org.aksw.iguana.cc.query.source.AbstractQuerySource; import org.aksw.iguana.cc.query.source.impl.FileLineQuerySource; import org.apache.commons.io.FileUtils; import org.apache.jena.ext.com.google.common.collect.Lists; @@ -73,9 +73,9 @@ public void removeFolder() throws IOException { @Test public void testQueryCreation() throws IOException { - QuerySource originalSource = getQuerySource(); + AbstractQuerySource originalSource = getQuerySource(); PatternHandler ph = new PatternHandler(getConfig(), originalSource); - QuerySource qs = ph.generateQuerySource(); + AbstractQuerySource qs = ph.generateQuerySource(); //check if folder exist this.dir/hashCode/ with |queries| files int hashcode = originalSource.hashCode(); @@ -103,7 +103,7 @@ public void testQueryCreation() throws IOException { @Test public void testCaching() throws IOException { - QuerySource originalSource = getQuerySource(); + AbstractQuerySource originalSource = getQuerySource(); PatternHandler ph = new PatternHandler(getConfig(), originalSource); ph.generateQuerySource(); @@ -135,7 +135,7 @@ private Map getConfig() { return config; } - private QuerySource getQuerySource() { + private AbstractQuerySource getQuerySource() { return new FileLineQuerySource(this.queriesFile.getAbsolutePath()); } } diff --git a/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/pattern/PatternHandlerTest.java b/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/pattern/PatternHandlerTest.java index c618815d9..d8f2b6581 100644 --- a/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/pattern/PatternHandlerTest.java +++ b/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/pattern/PatternHandlerTest.java @@ -1,6 +1,6 @@ package org.aksw.iguana.cc.query.pattern; -import org.aksw.iguana.cc.query.source.QuerySource; +import org.aksw.iguana.cc.query.source.AbstractQuerySource; import org.aksw.iguana.cc.query.source.impl.FileLineQuerySource; import org.aksw.iguana.cc.utils.ServerMock; import org.apache.jena.ext.com.google.common.collect.Lists; @@ -106,7 +106,7 @@ private PatternHandler getHandler() { config.put("endpoint", this.service); config.put("outputFolder", this.dir); - QuerySource qs = new FileLineQuerySource("src/test/resources/workers/single-query.txt"); + AbstractQuerySource qs = new FileLineQuerySource("src/test/resources/workers/single-query.txt"); return new PatternHandler(config, qs); } From 470b9385527d0abff2fd0d50559b0a0eac67e3fe Mon Sep 17 00:00:00 2001 From: nck-mlcnv <32801560+nck-mlcnv@users.noreply.github.com> Date: Wed, 5 Apr 2023 18:25:40 +0200 Subject: [PATCH 45/47] rename abstract classes --- .../org/aksw/iguana/cc/query/handler/QueryHandler.java | 10 +++++----- .../aksw/iguana/cc/query/pattern/PatternHandler.java | 8 ++++---- .../query/set/{AbstractQuerySet.java => QuerySet.java} | 8 ++++---- .../iguana/cc/query/set/impl/FileBasedQuerySet.java | 8 ++++---- .../aksw/iguana/cc/query/set/impl/InMemQuerySet.java | 8 ++++---- .../{AbstractQuerySource.java => QuerySource.java} | 4 ++-- .../cc/query/source/impl/FileLineQuerySource.java | 4 ++-- .../cc/query/source/impl/FileSeparatorQuerySource.java | 4 ++-- .../iguana/cc/query/source/impl/FolderQuerySource.java | 4 ++-- .../cc/query/pattern/PatternBasedQueryHandlerTest.java | 10 +++++----- .../iguana/cc/query/pattern/PatternHandlerTest.java | 4 ++-- 11 files changed, 36 insertions(+), 36 deletions(-) rename iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/{AbstractQuerySet.java => QuerySet.java} (85%) rename iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/{AbstractQuerySource.java => QuerySource.java} (93%) diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/handler/QueryHandler.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/handler/QueryHandler.java index 751e533bc..7cfed6a56 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/handler/QueryHandler.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/handler/QueryHandler.java @@ -6,10 +6,10 @@ import org.aksw.iguana.cc.query.selector.QuerySelector; import org.aksw.iguana.cc.query.selector.impl.LinearQuerySelector; import org.aksw.iguana.cc.query.selector.impl.RandomQuerySelector; -import org.aksw.iguana.cc.query.set.AbstractQuerySet; +import org.aksw.iguana.cc.query.set.QuerySet; import org.aksw.iguana.cc.query.set.impl.FileBasedQuerySet; import org.aksw.iguana.cc.query.set.impl.InMemQuerySet; -import org.aksw.iguana.cc.query.source.AbstractQuerySource; +import org.aksw.iguana.cc.query.source.QuerySource; import org.aksw.iguana.cc.query.source.impl.FileLineQuerySource; import org.aksw.iguana.cc.query.source.impl.FileSeparatorQuerySource; import org.aksw.iguana.cc.query.source.impl.FolderQuerySource; @@ -45,7 +45,7 @@ public class QueryHandler { protected QuerySelector querySelector; - protected AbstractQuerySet querySet; + protected QuerySet querySet; protected LanguageProcessor langProcessor; @@ -113,7 +113,7 @@ private void initPatternQuerySet() { * * @param querySource The QuerySource which contains the queries. */ - private void initQuerySet(AbstractQuerySource querySource) { + private void initQuerySet(QuerySource querySource) { this.caching = (Boolean) this.config.getOrDefault("caching", true); if (this.caching) { @@ -144,7 +144,7 @@ private void initQuerySet() { * * @return The QuerySource which contains the queries. */ - private AbstractQuerySource createQuerySource() { + private QuerySource createQuerySource() { Object formatObj = this.config.getOrDefault("format", "one-per-line"); if (formatObj instanceof Map) { Map format = (Map) formatObj; diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/pattern/PatternHandler.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/pattern/PatternHandler.java index 4d8392000..63cb1a907 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/pattern/PatternHandler.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/pattern/PatternHandler.java @@ -1,6 +1,6 @@ package org.aksw.iguana.cc.query.pattern; -import org.aksw.iguana.cc.query.source.AbstractQuerySource; +import org.aksw.iguana.cc.query.source.QuerySource; import org.aksw.iguana.cc.query.source.impl.FileLineQuerySource; import org.apache.jena.query.*; import org.slf4j.Logger; @@ -26,12 +26,12 @@ public class PatternHandler { private static final Logger LOGGER = LoggerFactory.getLogger(PatternHandler.class); private final Map config; - private final AbstractQuerySource querySource; + private final QuerySource querySource; private String endpoint; private Long limit; private String outputFolder; - public PatternHandler(Map config, AbstractQuerySource querySource) { + public PatternHandler(Map config, QuerySource querySource) { this.config = config; this.querySource = querySource; @@ -45,7 +45,7 @@ public PatternHandler(Map config, AbstractQuerySource querySourc * * @return QuerySource containing the instantiated queries */ - public AbstractQuerySource generateQuerySource() { + public QuerySource generateQuerySource() { File cacheFile = new File(this.outputFolder + File.separator + this.querySource.hashCode()); if (cacheFile.exists()) { diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/AbstractQuerySet.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/QuerySet.java similarity index 85% rename from iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/AbstractQuerySet.java rename to iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/QuerySet.java index a22690ef8..60de4523e 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/AbstractQuerySet.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/QuerySet.java @@ -1,6 +1,6 @@ package org.aksw.iguana.cc.query.set; -import org.aksw.iguana.cc.query.source.AbstractQuerySource; +import org.aksw.iguana.cc.query.source.QuerySource; import java.io.IOException; @@ -9,15 +9,15 @@ * * @author frensing */ -public abstract class AbstractQuerySet { +public abstract class QuerySet { /** This is the QuerySource from which the queries should be retrieved. */ - protected AbstractQuerySource querySource; + protected QuerySource querySource; /** A name for the query set. This is a part of the queryIDs. */ protected String name; - public AbstractQuerySet(String name, AbstractQuerySource querySource) { + public QuerySet(String name, QuerySource querySource) { this.name = name; this.querySource = querySource; } diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/impl/FileBasedQuerySet.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/impl/FileBasedQuerySet.java index 19a658927..83492e518 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/impl/FileBasedQuerySet.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/impl/FileBasedQuerySet.java @@ -1,7 +1,7 @@ package org.aksw.iguana.cc.query.set.impl; -import org.aksw.iguana.cc.query.set.AbstractQuerySet; -import org.aksw.iguana.cc.query.source.AbstractQuerySource; +import org.aksw.iguana.cc.query.set.QuerySet; +import org.aksw.iguana.cc.query.source.QuerySource; import java.io.IOException; @@ -10,9 +10,9 @@ * * @author frensing */ -public class FileBasedQuerySet extends AbstractQuerySet { +public class FileBasedQuerySet extends QuerySet { - public FileBasedQuerySet(String name, AbstractQuerySource querySource) { + public FileBasedQuerySet(String name, QuerySource querySource) { super(name, querySource); } diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/impl/InMemQuerySet.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/impl/InMemQuerySet.java index 0ff0a0177..3509047a9 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/impl/InMemQuerySet.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/impl/InMemQuerySet.java @@ -1,7 +1,7 @@ package org.aksw.iguana.cc.query.set.impl; -import org.aksw.iguana.cc.query.set.AbstractQuerySet; -import org.aksw.iguana.cc.query.source.AbstractQuerySource; +import org.aksw.iguana.cc.query.set.QuerySet; +import org.aksw.iguana.cc.query.source.QuerySource; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -14,13 +14,13 @@ * * @author frensing */ -public class InMemQuerySet extends AbstractQuerySet { +public class InMemQuerySet extends QuerySet { private static final Logger LOGGER = LoggerFactory.getLogger(InMemQuerySet.class); private List queries; - public InMemQuerySet(String name, AbstractQuerySource querySource) { + public InMemQuerySet(String name, QuerySource querySource) { super(name, querySource); loadQueries(); } diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/AbstractQuerySource.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/QuerySource.java similarity index 93% rename from iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/AbstractQuerySource.java rename to iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/QuerySource.java index 4c107c83b..e6835f24a 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/AbstractQuerySource.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/QuerySource.java @@ -11,12 +11,12 @@ * * @author frensing */ -public abstract class AbstractQuerySource { +public abstract class QuerySource { /** This string represents the path of the file or folder, that contains the queries. */ protected String path; - public AbstractQuerySource(String path) { + public QuerySource(String path) { this.path = path; } diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/impl/FileLineQuerySource.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/impl/FileLineQuerySource.java index 67b839597..80d7def76 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/impl/FileLineQuerySource.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/impl/FileLineQuerySource.java @@ -1,6 +1,6 @@ package org.aksw.iguana.cc.query.source.impl; -import org.aksw.iguana.cc.query.source.AbstractQuerySource; +import org.aksw.iguana.cc.query.source.QuerySource; import org.aksw.iguana.cc.utils.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -15,7 +15,7 @@ * * @author frensing */ -public class FileLineQuerySource extends AbstractQuerySource { +public class FileLineQuerySource extends QuerySource { private static final Logger LOGGER = LoggerFactory.getLogger(FileLineQuerySource.class); protected File queryFile; diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/impl/FileSeparatorQuerySource.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/impl/FileSeparatorQuerySource.java index f96604921..a7d36df7f 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/impl/FileSeparatorQuerySource.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/impl/FileSeparatorQuerySource.java @@ -1,6 +1,6 @@ package org.aksw.iguana.cc.query.source.impl; -import org.aksw.iguana.cc.query.source.AbstractQuerySource; +import org.aksw.iguana.cc.query.source.QuerySource; import org.aksw.iguana.cc.utils.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -21,7 +21,7 @@ * * @author frensing */ -public class FileSeparatorQuerySource extends AbstractQuerySource { +public class FileSeparatorQuerySource extends QuerySource { private static final Logger LOGGER = LoggerFactory.getLogger(FileSeparatorQuerySource.class); private static final String DEFAULT_SEPARATOR = "###"; diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/impl/FolderQuerySource.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/impl/FolderQuerySource.java index 0075b0047..3a3613fd1 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/impl/FolderQuerySource.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/impl/FolderQuerySource.java @@ -1,6 +1,6 @@ package org.aksw.iguana.cc.query.source.impl; -import org.aksw.iguana.cc.query.source.AbstractQuerySource; +import org.aksw.iguana.cc.query.source.QuerySource; import org.aksw.iguana.cc.utils.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -18,7 +18,7 @@ * * @author frensing */ -public class FolderQuerySource extends AbstractQuerySource { +public class FolderQuerySource extends QuerySource { protected static final Logger LOGGER = LoggerFactory.getLogger(FolderQuerySource.class); diff --git a/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/pattern/PatternBasedQueryHandlerTest.java b/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/pattern/PatternBasedQueryHandlerTest.java index 92a43d63c..2c082531c 100644 --- a/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/pattern/PatternBasedQueryHandlerTest.java +++ b/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/pattern/PatternBasedQueryHandlerTest.java @@ -1,6 +1,6 @@ package org.aksw.iguana.cc.query.pattern; -import org.aksw.iguana.cc.query.source.AbstractQuerySource; +import org.aksw.iguana.cc.query.source.QuerySource; import org.aksw.iguana.cc.query.source.impl.FileLineQuerySource; import org.apache.commons.io.FileUtils; import org.apache.jena.ext.com.google.common.collect.Lists; @@ -73,9 +73,9 @@ public void removeFolder() throws IOException { @Test public void testQueryCreation() throws IOException { - AbstractQuerySource originalSource = getQuerySource(); + QuerySource originalSource = getQuerySource(); PatternHandler ph = new PatternHandler(getConfig(), originalSource); - AbstractQuerySource qs = ph.generateQuerySource(); + QuerySource qs = ph.generateQuerySource(); //check if folder exist this.dir/hashCode/ with |queries| files int hashcode = originalSource.hashCode(); @@ -103,7 +103,7 @@ public void testQueryCreation() throws IOException { @Test public void testCaching() throws IOException { - AbstractQuerySource originalSource = getQuerySource(); + QuerySource originalSource = getQuerySource(); PatternHandler ph = new PatternHandler(getConfig(), originalSource); ph.generateQuerySource(); @@ -135,7 +135,7 @@ private Map getConfig() { return config; } - private AbstractQuerySource getQuerySource() { + private QuerySource getQuerySource() { return new FileLineQuerySource(this.queriesFile.getAbsolutePath()); } } diff --git a/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/pattern/PatternHandlerTest.java b/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/pattern/PatternHandlerTest.java index d8f2b6581..c618815d9 100644 --- a/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/pattern/PatternHandlerTest.java +++ b/iguana.corecontroller/src/test/java/org/aksw/iguana/cc/query/pattern/PatternHandlerTest.java @@ -1,6 +1,6 @@ package org.aksw.iguana.cc.query.pattern; -import org.aksw.iguana.cc.query.source.AbstractQuerySource; +import org.aksw.iguana.cc.query.source.QuerySource; import org.aksw.iguana.cc.query.source.impl.FileLineQuerySource; import org.aksw.iguana.cc.utils.ServerMock; import org.apache.jena.ext.com.google.common.collect.Lists; @@ -106,7 +106,7 @@ private PatternHandler getHandler() { config.put("endpoint", this.service); config.put("outputFolder", this.dir); - AbstractQuerySource qs = new FileLineQuerySource("src/test/resources/workers/single-query.txt"); + QuerySource qs = new FileLineQuerySource("src/test/resources/workers/single-query.txt"); return new PatternHandler(config, qs); } From 9bf549d9d88c2e9d3375510726a88e0f8282031e Mon Sep 17 00:00:00 2001 From: nck-mlcnv <32801560+nck-mlcnv@users.noreply.github.com> Date: Wed, 5 Apr 2023 19:18:36 +0200 Subject: [PATCH 46/47] rename QuerySet to QueryList and the method getQueryAtPos to getQuery --- .../iguana/cc/query/handler/QueryHandler.java | 46 +++++++------- .../aksw/iguana/cc/query/list/QueryList.java | 60 +++++++++++++++++++ .../query/list/impl/FileBasedQueryList.java | 23 +++++++ .../impl/InMemQueryList.java} | 16 ++--- .../aksw/iguana/cc/query/set/QuerySet.java | 60 ------------------- .../cc/query/set/impl/FileBasedQuerySet.java | 23 ------- .../iguana/cc/query/source/QuerySource.java | 2 +- 7 files changed, 115 insertions(+), 115 deletions(-) create mode 100644 iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/list/QueryList.java create mode 100644 iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/list/impl/FileBasedQueryList.java rename iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/{set/impl/InMemQuerySet.java => list/impl/InMemQueryList.java} (65%) delete mode 100644 iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/QuerySet.java delete mode 100644 iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/impl/FileBasedQuerySet.java diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/handler/QueryHandler.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/handler/QueryHandler.java index 7cfed6a56..b6e72e142 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/handler/QueryHandler.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/handler/QueryHandler.java @@ -6,9 +6,9 @@ import org.aksw.iguana.cc.query.selector.QuerySelector; import org.aksw.iguana.cc.query.selector.impl.LinearQuerySelector; import org.aksw.iguana.cc.query.selector.impl.RandomQuerySelector; -import org.aksw.iguana.cc.query.set.QuerySet; -import org.aksw.iguana.cc.query.set.impl.FileBasedQuerySet; -import org.aksw.iguana.cc.query.set.impl.InMemQuerySet; +import org.aksw.iguana.cc.query.list.QueryList; +import org.aksw.iguana.cc.query.list.impl.FileBasedQueryList; +import org.aksw.iguana.cc.query.list.impl.InMemQueryList; import org.aksw.iguana.cc.query.source.QuerySource; import org.aksw.iguana.cc.query.source.impl.FileLineQuerySource; import org.aksw.iguana.cc.query.source.impl.FileSeparatorQuerySource; @@ -26,7 +26,7 @@ /** * The QueryHandler is used by every worker that extends the AbstractWorker. - * It initializes the QuerySource, QuerySelector, QuerySet and, if needed, PatternHandler. + * It initializes the QuerySource, QuerySelector, QueryList and, if needed, PatternHandler. * After the initialization, it provides the next query to the worker using the generated QuerySource * and the order given by the QuerySelector. * @@ -45,7 +45,7 @@ public class QueryHandler { protected QuerySelector querySelector; - protected QuerySet querySet; + protected QueryList queryList; protected LanguageProcessor langProcessor; @@ -60,22 +60,22 @@ public QueryHandler(Map config, Integer workerID) { initQuerySelector(); initLanguageProcessor(); - this.hashcode = this.querySet.hashCode(); + this.hashcode = this.queryList.hashCode(); } public void getNextQuery(StringBuilder queryStr, StringBuilder queryID) throws IOException { int queryIndex = this.querySelector.getNextIndex(); - queryStr.append(this.querySet.getQueryAtPos(queryIndex)); + queryStr.append(this.queryList.getQuery(queryIndex)); queryID.append(getQueryId(queryIndex)); } public Model getTripleStats(String taskID) { - List queries = new ArrayList<>(this.querySet.size()); - for (int i = 0; i < this.querySet.size(); i++) { + List queries = new ArrayList<>(this.queryList.size()); + for (int i = 0; i < this.queryList.size(); i++) { try { - queries.add(new QueryWrapper(this.querySet.getQueryAtPos(i), getQueryId(i))); + queries.add(new QueryWrapper(this.queryList.getQuery(i), getQueryId(i))); } catch (Exception e) { - LOGGER.error("Could not parse query " + this.querySet.getName() + ":" + i, e); + LOGGER.error("Could not parse query " + this.queryList.getName() + ":" + i, e); } } return this.langProcessor.generateTripleStats(queries, "" + this.hashcode, taskID); @@ -87,7 +87,7 @@ public int hashCode() { } public int getQueryCount() { - return this.querySet.size(); + return this.queryList.size(); } public LanguageProcessor getLanguageProcessor() { @@ -97,7 +97,7 @@ public LanguageProcessor getLanguageProcessor() { /** * This method initializes the PatternHandler if a pattern config is given, therefore * this.config.get("pattern") should return an appropriate pattern configuration and not - * null. The PatternHandler uses the original query source to generate a new query source and set with + * null. The PatternHandler uses the original query source to generate a new query source and list with * the instantiated queries. */ private void initPatternQuerySet() { @@ -108,8 +108,8 @@ private void initPatternQuerySet() { } /** - * Will initialize the QuerySet. - * If caching is not set or set to true, the InMemQuerySet will be used. Otherwise the FileBasedQuerySet. + * Will initialize the QueryList. + * If caching is not set or set to true, the InMemQueryList will be used. Otherwise the FileBasedQueryList. * * @param querySource The QuerySource which contains the queries. */ @@ -117,15 +117,15 @@ private void initQuerySet(QuerySource querySource) { this.caching = (Boolean) this.config.getOrDefault("caching", true); if (this.caching) { - this.querySet = new InMemQuerySet(this.location, querySource); + this.queryList = new InMemQueryList(this.location, querySource); } else { - this.querySet = new FileBasedQuerySet(this.location, querySource); + this.queryList = new FileBasedQueryList(this.location, querySource); } } /** - * This method initializes the QuerySet for the QueryHandler. If a pattern configuration is specified, this method - * will execute initPatternQuerySet to create the QuerySet. + * This method initializes the QueryList for the QueryHandler. If a pattern configuration is specified, this method + * will execute initPatternQuerySet to create the QueryList. */ private void initQuerySet() { if(this.config.containsKey("pattern")) { @@ -176,11 +176,11 @@ private void initQuerySelector() { if (orderObj instanceof String) { String order = (String) orderObj; if (order.equals("linear")) { - this.querySelector = new LinearQuerySelector(this.querySet.size()); + this.querySelector = new LinearQuerySelector(this.queryList.size()); return; } if (order.equals("random")) { - this.querySelector = new RandomQuerySelector(this.querySet.size(), this.workerID); + this.querySelector = new RandomQuerySelector(this.queryList.size(), this.workerID); return; } @@ -191,7 +191,7 @@ private void initQuerySelector() { if (order.containsKey("random")) { Map random = (Map) order.get("random"); Integer seed = (Integer) random.get("seed"); - this.querySelector = new RandomQuerySelector(this.querySet.size(), seed); + this.querySelector = new RandomQuerySelector(this.queryList.size(), seed); return; } LOGGER.error("Unknown order: " + order); @@ -208,6 +208,6 @@ private void initLanguageProcessor() { } private String getQueryId(int i) { - return this.querySet.getName() + ":" + i; + return this.queryList.getName() + ":" + i; } } diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/list/QueryList.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/list/QueryList.java new file mode 100644 index 000000000..39b0961cb --- /dev/null +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/list/QueryList.java @@ -0,0 +1,60 @@ +package org.aksw.iguana.cc.query.list; + +import org.aksw.iguana.cc.query.source.QuerySource; + +import java.io.IOException; + +/** + * The abstract class for a QueryList. A query list provides the queries to the QueryHandler. + * + * @author frensing + */ +public abstract class QueryList { + + /** This is the QuerySource from which the queries should be retrieved. */ + protected QuerySource querySource; + + /** A name for the query list. This is a part of the queryIDs. */ + protected String name; + + public QueryList(String name, QuerySource querySource) { + this.name = name; + this.querySource = querySource; + } + + /** + * This method returns the amount of queries in the query list. + * + * @return The amount of queries in the query list + */ + public int size() { + return this.querySource.size(); + } + + /** + * This method returns the name of the query list. + * + * @return The name of the query list + */ + public String getName() { + return this.name; + } + + /** + * This method returns the hashcode of the query list which is the hashcode of the query source. + * + * @return The hashcode of the query list + */ + @Override + public int hashCode() { + return this.querySource.hashCode(); + } + + /** + * This method returns a query at the given index. + * + * @param index Index of the query in the list + * @return The query at the given index + */ + public abstract String getQuery(int index) throws IOException; +} diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/list/impl/FileBasedQueryList.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/list/impl/FileBasedQueryList.java new file mode 100644 index 000000000..76d1ef459 --- /dev/null +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/list/impl/FileBasedQueryList.java @@ -0,0 +1,23 @@ +package org.aksw.iguana.cc.query.list.impl; + +import org.aksw.iguana.cc.query.list.QueryList; +import org.aksw.iguana.cc.query.source.QuerySource; + +import java.io.IOException; + +/** + * A query list which reads the queries directly from a file. + * + * @author frensing + */ +public class FileBasedQueryList extends QueryList { + + public FileBasedQueryList(String name, QuerySource querySource) { + super(name, querySource); + } + + @Override + public String getQuery(int index) throws IOException { + return this.querySource.getQuery(index); + } +} diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/impl/InMemQuerySet.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/list/impl/InMemQueryList.java similarity index 65% rename from iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/impl/InMemQuerySet.java rename to iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/list/impl/InMemQueryList.java index 3509047a9..d2cfb86b3 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/impl/InMemQuerySet.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/list/impl/InMemQueryList.java @@ -1,6 +1,6 @@ -package org.aksw.iguana.cc.query.set.impl; +package org.aksw.iguana.cc.query.list.impl; -import org.aksw.iguana.cc.query.set.QuerySet; +import org.aksw.iguana.cc.query.list.QueryList; import org.aksw.iguana.cc.query.source.QuerySource; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -9,18 +9,18 @@ import java.util.List; /** - * A query set which reads the queries into memory on initialization. + * A query list which reads the queries into memory on initialization. * During the benchmark the query are returned from the memory. * * @author frensing */ -public class InMemQuerySet extends QuerySet { +public class InMemQueryList extends QueryList { - private static final Logger LOGGER = LoggerFactory.getLogger(InMemQuerySet.class); + private static final Logger LOGGER = LoggerFactory.getLogger(InMemQueryList.class); private List queries; - public InMemQuerySet(String name, QuerySource querySource) { + public InMemQueryList(String name, QuerySource querySource) { super(name, querySource); loadQueries(); } @@ -34,8 +34,8 @@ private void loadQueries() { } @Override - public String getQueryAtPos(int pos) { - return this.queries.get(pos); + public String getQuery(int index) { + return this.queries.get(index); } @Override diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/QuerySet.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/QuerySet.java deleted file mode 100644 index 60de4523e..000000000 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/QuerySet.java +++ /dev/null @@ -1,60 +0,0 @@ -package org.aksw.iguana.cc.query.set; - -import org.aksw.iguana.cc.query.source.QuerySource; - -import java.io.IOException; - -/** - * The abstract class for a QuerySet. A query set provides the queries to the QueryHandler. - * - * @author frensing - */ -public abstract class QuerySet { - - /** This is the QuerySource from which the queries should be retrieved. */ - protected QuerySource querySource; - - /** A name for the query set. This is a part of the queryIDs. */ - protected String name; - - public QuerySet(String name, QuerySource querySource) { - this.name = name; - this.querySource = querySource; - } - - /** - * This method returns the amount of queries in the query set. - * - * @return The amount of queries in the query set - */ - public int size() { - return this.querySource.size(); - } - - /** - * This method returns the name of the query set. - * - * @return The name of the query set - */ - public String getName() { - return this.name; - } - - /** - * This method returns the hashcode of the query set which is the hashcode of the query source. - * - * @return The hashcode of the query set - */ - @Override - public int hashCode() { - return this.querySource.hashCode(); - } - - /** - * This method returns a query at the given position. - * - * @param pos Position of the query in the set - * @return The query at position pos - */ - public abstract String getQueryAtPos(int pos) throws IOException; -} diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/impl/FileBasedQuerySet.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/impl/FileBasedQuerySet.java deleted file mode 100644 index 83492e518..000000000 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/set/impl/FileBasedQuerySet.java +++ /dev/null @@ -1,23 +0,0 @@ -package org.aksw.iguana.cc.query.set.impl; - -import org.aksw.iguana.cc.query.set.QuerySet; -import org.aksw.iguana.cc.query.source.QuerySource; - -import java.io.IOException; - -/** - * A query set which reads the queries directly from a file. - * - * @author frensing - */ -public class FileBasedQuerySet extends QuerySet { - - public FileBasedQuerySet(String name, QuerySource querySource) { - super(name, querySource); - } - - @Override - public String getQueryAtPos(int pos) throws IOException { - return this.querySource.getQuery(pos); - } -} diff --git a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/QuerySource.java b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/QuerySource.java index e6835f24a..f505a8da6 100644 --- a/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/QuerySource.java +++ b/iguana.corecontroller/src/main/java/org/aksw/iguana/cc/query/source/QuerySource.java @@ -7,7 +7,7 @@ /** * The abstract class for a QuerySource.
- * The QuerySource provides the queries to the QuerySet. It abstracts the actual format of the query files. + * The QuerySource provides the queries to the QueryList. It abstracts the actual format of the query files. * * @author frensing */ From 92498c8e84cfeac9a4c799e5f355780f5763f6ac Mon Sep 17 00:00:00 2001 From: nck-mlcnv <32801560+nck-mlcnv@users.noreply.github.com> Date: Fri, 7 Apr 2023 20:35:50 +0200 Subject: [PATCH 47/47] update the query handling development doc page --- docs/develop/extend-queryhandling.md | 39 ++++++++++++++-------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/docs/develop/extend-queryhandling.md b/docs/develop/extend-queryhandling.md index 3805961b6..f4bb6150f 100644 --- a/docs/develop/extend-queryhandling.md +++ b/docs/develop/extend-queryhandling.md @@ -2,25 +2,23 @@ Currently, there is no way of extending the query handling without modifying the QueryHandler class. -You can change the way queries are handled by extending the following abstract classes: +You can change the way queries are handled by extending the following abstract class and interface: -| Class | Function | -|-------------------------|------------------------------------------------------------------| -| `AbstractQuerySelector` | Responsible for selecting the next query a worker should execute | -| `AbstractQuerySource` | Responsible for loading queries | +| Class | Function | +|-----------------|------------------------------------------------------------------| +| `QuerySelector` | Responsible for selecting the next query a worker should execute | +| `QuerySource` | Responsible for loading queries | -In the following sections, each extension of a class will be described briefly with the necessary changes to the `QueryHandler` class. For further details, read the corresponding javadocs. +In the following sections, each extension of a class and implementation will be described briefly with the necessary changes to the +`QueryHandler` class. For further details, read the corresponding javadocs. ## QuerySelector -If you want a different execution order for your queries, you can create a class that extends the class `AbstractQuerySelector` and implements the method `getNextIndex`: +If you want a different execution order for your queries, you can create a class that implements the interface +`QuerySelector` and the method `getNextIndex`: ```java -public class MyQuerySelector extends AbstractQuerySelector { - public MyQuerySelector(int size) { - super(size); - } - +public class MyQuerySelector implements QuerySelector { @Override public int getNextIndex(){ // code for selecting the next query a worker should execute @@ -28,7 +26,8 @@ public class MyQuerySelector extends AbstractQuerySelector { } ``` -Once you've created your QuerySelector class, you need to decide a value for the key `order` (in this example `"myOrder"`) for the configuration file and update the `initQuerySelector` method inside the `QueryHandler` class: +Once you've created your QuerySelector class, you need to decide a value for the key `order` (in this example +`"myOrder"`) for the configuration file and update the `initQuerySelector` method inside the `QueryHandler` class: ```java private void initQuerySelector() { @@ -37,17 +36,17 @@ private void initQuerySelector() { if (orderObj instanceof String) { String order = (String) orderObj; if (order.equals("linear")) { - this.querySelector = new LinearQuerySelector(this.querySet.size()); + this.querySelector = new LinearQuerySelector(this.queryList.size()); return; } if (order.equals("random")) { - this.querySelector = new RandomQuerySelector(this.querySet.size(), this.workerID); + this.querySelector = new RandomQuerySelector(this.queryList.size(), this.workerID); return; } // add this if (order.equals("myOrder")) { - this.querySelector = new MyQuerySelector(this.querySet.size(), this.workerID); + this.querySelector = new MyQuerySelector(); return; } @@ -60,10 +59,11 @@ private void initQuerySelector() { ## QuerySource -If you want to use different source for your queries, you can create a class that extends the class `AbstractQuerySourcer` and implements the following methods: +If you want to use different source for your queries, you can create a class that extends the class `QuerySourcer` and +implements the following methods: ```java -public class MyQuerySource extends AbstractQuerySource { +public class MyQuerySource extends QuerySource { public MyQuerySource(String filepath) { // your constructor // filepath is the value, specified in the "location"-key inside the configuration file @@ -85,7 +85,8 @@ public class MyQuerySource extends AbstractQuerySource { } } ``` -Once you have created your QuerySelector class, you need to decide a value for the key `format` (in this example `"myFormat"`) for the configuration file and update the `createQuerySource` method inside the `QueryHandler` class: +Once you have created your QuerySelector class, you need to decide a value for the key `format` (in this example +`"myFormat"`) for the configuration file and update the `createQuerySource` method inside the `QueryHandler` class: ```Java private QuerySource createQuerySource() {