Skip to content

Commit

Permalink
match up making and parsing doc-ids
Browse files Browse the repository at this point in the history
  • Loading branch information
PJ committed Dec 11, 2014
1 parent 637594b commit 06ffba9
Show file tree
Hide file tree
Showing 2 changed files with 149 additions and 25 deletions.
43 changes: 38 additions & 5 deletions src/com/google/enterprise/adaptor/database/UniqueKey.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import java.sql.ResultSet;
import java.sql.SQLException;

import java.util.Arrays;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
Expand Down Expand Up @@ -166,14 +167,17 @@ String makeUniqueId(ResultSet rs) throws SQLException {

private void setSqlValues(PreparedStatement st, String uniqueId,
List<String> sqlCols) throws SQLException {
uniqueId = decodeSlashInData(uniqueId);
String parts[] = uniqueId.split("/", 0);
// parse on / that isn't preceded by \
// because a / that is preceded by \ is part of column value
String parts[] = uniqueId.split("(?<!\\\\)/", -1);
if (parts.length != names.size()) {
throw new IllegalStateException("wrong number of values for primary key");
throw new IllegalStateException(
"wrong number of values for primary key: "
+ "id: " + uniqueId + ", parts: " + Arrays.asList(parts));
}
Map<String, String> zip = new TreeMap<String, String>();
for (int i = 0; i < parts.length; i++) {
zip.put(names.get(i), parts[i]);
zip.put(names.get(i), decodeSlashInData(parts[i]));
}
for (int i = 0; i < sqlCols.size(); i++) {
String colName = sqlCols.get(i);
Expand Down Expand Up @@ -217,16 +221,45 @@ void setAclSqlValues(PreparedStatement st, String uniqueId)

@VisibleForTesting
static String encodeSlashInData(String data) {
// TODO: change escape character from \ to some other character
// because browsers convert \ to /
if (-1 == data.indexOf('/') && -1 == data.indexOf('\\')) {
return data;
}
char lastChar = data.charAt(data.length() - 1);
// Don't let data end with \, because then a \ would
// precede the seperator /. If column value ends with
// \ then append a / and take it away when decoding.
// Since, data could also end with / that we wouldn't
// want taken away during decoding, append a second /
// when data ends with / too.

if ('\\' == lastChar || '/' == lastChar) {
// For \ case:
// Suppose unique key values are 5\ and 6\. Without this code here
// the unique keys would be encoded into DocId "5\\/6\\". Then when
// parsing DocId, the slash would not be used as a // splitter because
// it preceded by \.
// For / case:
// Suppose unique key values are 5/ and 6/. Without appending another
// /, DocId will be 5\//6\/, which will be split and decoded as 5
// and 6.
data += '/';
}
data = data.replace("\\", "\\\\");
data = data.replace("/", "\\/");
return data;
return data;
}

@VisibleForTesting
static String decodeSlashInData(String id) {
// If column value ends with / (encoded as \/)
// we know that it was appended because colulmn
// value ended with either \ or /. We take away
// this last added / character.
if (id.endsWith("\\/")) {
id = id.substring(0, id.length() - 2);
}
id = id.replace("\\/", "/");
id = id.replace("\\\\", "\\");
return id;
Expand Down
131 changes: 111 additions & 20 deletions test/com/google/enterprise/adaptor/database/UniqueKeyTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,9 @@
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Random;

/** Test cases for {@link UniqueKey}. */
Expand Down Expand Up @@ -117,16 +119,16 @@ public void testUnknownAclCol() {
}

/* Proxy based mock of ResultSet, because it has lots of methods to mock. */
private ResultSet makeMockResultSet(final int n, final String s) {
private ResultSet makeMockResultSet(final Map<String, Object> columnValueMap) {
ResultSet rs = (ResultSet) Proxy.newProxyInstance(
ResultSet.class.getClassLoader(), new Class[] { ResultSet.class },
new InvocationHandler() {
public Object invoke(Object proxy, Method method, Object[] args)
throws Throwable {
switch (args[0].toString()) {
case "numnum": return n;
case "strstr": return s;
default: throw new IllegalStateException();
if (columnValueMap.containsKey(args[0])) {
return columnValueMap.get(args[0]);
} else {
throw new IllegalStateException();
}
}
}
Expand All @@ -137,7 +139,36 @@ public Object invoke(Object proxy, Method method, Object[] args)
@Test
public void testProcessingDocId() throws SQLException {
UniqueKey uk = new UniqueKey("numnum:int,strstr:string");
assertEquals("345/abc", uk.makeUniqueId(makeMockResultSet(345, "abc")));
assertEquals("345/abc", uk.makeUniqueId(makeMockResultSet(
new HashMap<String, Object>(){{
put("numnum", 345);
put("strstr", "abc");
}}
)));
}

@Test
public void testProcessingDocIdWithSlash() throws SQLException {
UniqueKey uk = new UniqueKey("a:string,b:string");
assertEquals("5\\/5/6\\/6", uk.makeUniqueId(makeMockResultSet(
new HashMap<String, Object>(){{
put("a", "5/5");
put("b", "6/6");
}}
)));
}

@Test
public void testProcessingDocIdWithMoreSlashes() throws SQLException {
UniqueKey uk = new UniqueKey("a:string,b:string");
assertEquals("5\\/5\\/\\/\\//\\/\\/6\\/6",
uk.makeUniqueId(makeMockResultSet(
new HashMap<String, Object>(){{
put("a", "5/5//");
put("b", "//6/6");
}}
)
));
}

private static class PreparedStatementHandler implements
Expand Down Expand Up @@ -178,7 +209,6 @@ public void testPreparingRetrieval() throws SQLException {
PreparedStatement ps = (PreparedStatement) Proxy.newProxyInstance(
PreparedStatement.class.getClassLoader(),
new Class[] { PreparedStatement.class }, psh);

UniqueKey uk = new UniqueKey("numnum:int,strstr:string,"
+ "timestamp:timestamp,date:date,time:time,long:long");
uk.setContentSqlValues(ps, "888/bluesky/1414701070212/2014-01-01/"
Expand Down Expand Up @@ -206,7 +236,6 @@ public void testPreparingRetrievalPerDocCols() throws SQLException {
PreparedStatement ps = (PreparedStatement) Proxy.newProxyInstance(
PreparedStatement.class.getClassLoader(),
new Class[] { PreparedStatement.class }, psh);

UniqueKey uk = new UniqueKey("numnum:int,strstr:string",
"numnum,numnum,strstr,numnum,strstr,strstr,numnum", "");
uk.setContentSqlValues(ps, "888/bluesky");
Expand All @@ -222,6 +251,79 @@ public void testPreparingRetrievalPerDocCols() throws SQLException {
assertEquals(golden, psh.callsAndValues);
}

@Test
public void testPreserveSlashesInColumnValues() throws SQLException {
PreparedStatementHandlerPerDocCols psh
= new PreparedStatementHandlerPerDocCols();
PreparedStatement ps = (PreparedStatement) Proxy.newProxyInstance(
PreparedStatement.class.getClassLoader(),
new Class[] { PreparedStatement.class }, psh);
UniqueKey uk = new UniqueKey("a:string,b:string",
"a,b,a", "");
uk.setContentSqlValues(ps, "5\\/5/6\\/6");
List<String> golden = Arrays.asList(
"setString", "1", "5/5",
"setString", "2", "6/6",
"setString", "3", "5/5"
);
assertEquals(golden, psh.callsAndValues);
}

private static String makeId(char choices[], int maxlen) {
StringBuilder sb = new StringBuilder();
Random r = new Random();
int len = r.nextInt(maxlen);
for (int i = 0; i < len; i++) {
sb.append(choices[r.nextInt(choices.length)]);
}
return "" + sb;
}

private static String makeRandomId() {
char choices[] = "13/\\45\\97%^&%$^)*(/<>|P{UITY*c".toCharArray();
return makeId(choices, 100);
}

private static String makeSomeSlashes() {
return makeId("/\\".toCharArray(), 100);
}

@Test
public void testFuzzSlashes() throws SQLException {
for (int fuzzCase = 0; fuzzCase < 1000; fuzzCase++) {
UniqueKey uk = new UniqueKey("a:string,b:string");
final String elem1 = makeSomeSlashes();
final String elem2 = makeSomeSlashes();
String id = uk.makeUniqueId(makeMockResultSet(
new HashMap<String, Object>(){{
put("a", elem1);
put("b", elem2);
}}
));
PreparedStatementHandlerPerDocCols psh
= new PreparedStatementHandlerPerDocCols();
PreparedStatement ps = (PreparedStatement) Proxy.newProxyInstance(
PreparedStatement.class.getClassLoader(),
new Class[] { PreparedStatement.class }, psh);
try {
uk.setContentSqlValues(ps, id);
} catch (Exception e) {
throw new RuntimeException("elem1: " + elem1 + ", elem2: " + elem2
+ ", id: " + id, e);
}
List<String> golden = Arrays.asList(
"setString", "1", elem1,
"setString", "2", elem2
);
try {
assertEquals(golden, psh.callsAndValues);
} catch (Throwable e) {
throw new RuntimeException("elem1: " + elem1 + ", elem2: " + elem2
+ ", id: " + id + ", golden: " + golden, e);
}
}
}

private static String roundtrip(String in) {
return UniqueKey.decodeSlashInData(UniqueKey.encodeSlashInData(in));
}
Expand Down Expand Up @@ -268,19 +370,8 @@ public void testWithMessOfSlashAndBackslash() {
assertEquals(id, roundtrip(id));
}

private static String makeRandomId() {
char choices[] = "13/\\45\\97%^&%$^)*(/<>|P{UITY*c".toCharArray();
StringBuilder sb = new StringBuilder();
Random r = new Random();
int len = r.nextInt(100);
for (int i = 0; i < len; i++) {
sb.append(choices[r.nextInt(choices.length)]);
}
return "" + sb;
}

@Test
public void testWithFuzz() {
public void testFuzzEncodeAndDecode() {
int ntests = 100;
for (int i = 0; i < ntests; i++) {
String fuzz = makeRandomId();
Expand Down

0 comments on commit 06ffba9

Please sign in to comment.