Skip to content
This repository has been archived by the owner on Dec 18, 2021. It is now read-only.

Commit

Permalink
removing ObjectMapper due to ES 2.2 security
Browse files Browse the repository at this point in the history
  • Loading branch information
@jprante
jprante committed Mar 6, 2016
1 parent 1825471 commit b2045c1
Show file tree
Hide file tree
Showing 4 changed files with 56 additions and 44 deletions.
43 changes: 27 additions & 16 deletions src/main/java/org/xbib/elasticsearch/common/langdetect/LangProfile.java
View file
Original file line number Diff line number Diff line change
@@ -1,24 +1,30 @@
package org.xbib.elasticsearch.common.langdetect;

import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentType;

import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

/**
* This class is used by ObjectMapper, it requires public attributes
*/
public class LangProfile {

public String name = null;
private String name;

public Map<String, Integer> freq = new HashMap<String, Integer>();
private Map<String, Integer> freq;

public int[] n_words = new int[NGram.N_GRAM];
private List<Integer> n_words;

public LangProfile() {
}

public LangProfile(String name) {
this.name = name;
this.freq = new HashMap<>();
this.n_words = new ArrayList<>(NGram.N_GRAM);
for (int i = 0; i < NGram.N_GRAM; i++) {
n_words.add(0);
}
}

public void add(String gram) {
Expand All @@ -29,7 +35,7 @@ public void add(String gram) {
if (len < 1 || len > NGram.N_GRAM) {
return;
}
++n_words[len - 1];
n_words.set(len - 1, n_words.get(len -1) + 1);
if (freq.containsKey(gram)) {
freq.put(gram, freq.get(gram) + 1);
} else {
Expand All @@ -45,11 +51,7 @@ public String getName() {
return name;
}

public void setNWords() {
this.n_words = n_words;
}

public int[] getNWords() {
public List<Integer> getNWords() {
return n_words;
}

Expand All @@ -61,4 +63,13 @@ public Map<String, Integer> getFreq() {
return freq;
}

@SuppressWarnings("unchecked")
public void read(InputStream input) throws IOException {
XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(input);
Map<String,Object> map = parser.map();
freq = (Map<String, Integer>) map.get("freq");
name = (String)map.get("name");
n_words = (List<Integer>)map.get("n_words");
}

}
14 changes: 4 additions & 10 deletions src/main/java/org/xbib/elasticsearch/module/langdetect/LangdetectService.java
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
package org.xbib.elasticsearch.module.langdetect;

import com.fasterxml.jackson.databind.ObjectMapper;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.common.component.AbstractLifecycleComponent;
import org.elasticsearch.common.inject.Inject;
Expand All @@ -14,12 +13,7 @@

import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.*;
import java.util.regex.Pattern;

public class LangdetectService extends AbstractLifecycleComponent<LangdetectService> {
Expand Down Expand Up @@ -195,8 +189,8 @@ public void loadProfileFromResource(String resource, String profile, int index,
if (in == null) {
throw new IOException("profile '" + resource + "' not found");
}
ObjectMapper mapper = new ObjectMapper();
LangProfile langProfile = mapper.readValue(in, LangProfile.class);
LangProfile langProfile = new LangProfile();
langProfile.read(in);
addProfile(langProfile, index, langsize);
}

Expand All @@ -212,7 +206,7 @@ public void addProfile(LangProfile profile, int index, int langsize) throws IOEx
}
int length = word.length();
if (length >= 1 && length <= 3) {
double prob = profile.getFreq().get(word).doubleValue() / profile.getNWords()[length - 1];
double prob = profile.getFreq().get(word).doubleValue() / profile.getNWords().get(length - 1);
wordLangProbMap.get(word)[index] = prob;
}
}
Expand Down
9 changes: 6 additions & 3 deletions src/test/java/org/xbib/elasticsearch/index/mapper/langdetect/DetectorTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,19 +28,22 @@ public static void setUp() throws Exception {
detect = new LangdetectService(settings);
detect.start();

LangProfile profile_en = new LangProfile("en");
LangProfile profile_en = new LangProfile();
profile_en.setName("en");
for (String w : TRAINING_EN.split(" ")) {
profile_en.add(w);
}
detect.addProfile(profile_en, 0, 3);

LangProfile profile_fr = new LangProfile("fr");
LangProfile profile_fr = new LangProfile();
profile_fr.setName("fr");
for (String w : TRAINING_FR.split(" ")) {
profile_fr.add(w);
}
detect.addProfile(profile_fr, 1, 3);

LangProfile profile_ja = new LangProfile("ja");
LangProfile profile_ja = new LangProfile();
profile_ja.setName("ja");
for (String w : TRAINING_JA.split(" ")) {
profile_ja.add(w);
}
Expand Down
34 changes: 19 additions & 15 deletions src/test/java/org/xbib/elasticsearch/index/mapper/langdetect/LangProfileTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,47 +9,51 @@ public class LangProfileTest extends Assert {
@Test
public final void testLangProfile() {
LangProfile profile = new LangProfile();
assertEquals(profile.name, null);
assertEquals(profile.getName(), null);
}

@Test
public final void testLangProfileStringInt() {
LangProfile profile = new LangProfile("en");
assertEquals(profile.name, "en");
LangProfile profile = new LangProfile();
profile.setName("en");
assertEquals(profile.getName(), "en");
}

@Test
public final void testAdd() {
LangProfile profile = new LangProfile("en");
LangProfile profile = new LangProfile();
profile.setName("en");
profile.add("a");
assertEquals((int) profile.freq.get("a"), 1);
assertEquals((int) profile.getFreq().get("a"), 1);
profile.add("a");
assertEquals((int) profile.freq.get("a"), 2);
assertEquals((int) profile.getFreq().get("a"), 2);
//profile.omitLessFreq();
}

@Test
public final void testAddIllegally1() {
LangProfile profile = new LangProfile();
profile.add("a");
assertEquals(profile.freq.get("a"), null);
assertEquals(profile.getFreq().get("a"), null);
}

@Test
public final void testAddIllegally2() {
LangProfile profile = new LangProfile("en");
LangProfile profile = new LangProfile();
profile.setName("en");
profile.add("a");
profile.add("");
profile.add("abcd");
assertEquals((int) profile.freq.get("a"), 1);
assertEquals(profile.freq.get(""), null);
assertEquals(profile.freq.get("abcd"), null);
assertEquals((int) profile.getFreq().get("a"), 1);
assertEquals(profile.getFreq().get(""), null);
assertEquals(profile.getFreq().get("abcd"), null);

}

@Test
public final void testOmitLessFreq() {
LangProfile profile = new LangProfile("en");
LangProfile profile = new LangProfile();
profile.setName("en");
String[] grams = "a b c \u3042 \u3044 \u3046 \u3048 \u304a \u304b \u304c \u304d \u304e \u304f".split(" ");
for (int i = 0; i < 5; ++i) {
for (String g : grams) {
Expand All @@ -58,9 +62,9 @@ public final void testOmitLessFreq() {
}
profile.add("\u3050");

assertEquals((int) profile.freq.get("a"), 5);
assertEquals((int) profile.freq.get("\u3042"), 5);
assertEquals((int) profile.freq.get("\u3050"), 1);
assertEquals((int) profile.getFreq().get("a"), 5);
assertEquals((int) profile.getFreq().get("\u3042"), 5);
assertEquals((int) profile.getFreq().get("\u3050"), 1);
//profile.omitLessFreq();
//assertEquals(profile.freq.get("a"), null);
//assertEquals((int) profile.freq.get("\u3042"), 5);
Expand Down

0 comments on commit b2045c1

Please sign in to comment.