Skip to content

Commit

Permalink
Merge branch 'master' into sig
Browse files Browse the repository at this point in the history
  • Loading branch information
@wonlay
wonlay committed Oct 10, 2012
2 parents 0299d0e + f18bcf3 commit 23218ac
Show file tree
Hide file tree
Showing 5 changed files with 77 additions and 16 deletions.
10 changes: 8 additions & 2 deletions sensei-core/src/main/java/com/senseidb/search/node/ResultMerger.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -101,10 +101,16 @@ SenseiHit getSenseiHit(SenseiRequest req)
Collection<FacetHandler<?>> facetHandlers= reader.getFacetHandlerMap().values();
Map<String,String[]> map = new HashMap<String,String[]>();
Map<String,Object[]> rawMap = new HashMap<String,Object[]>();
Set<String> selectSet = req.getSelectSet();
for (FacetHandler<?> facetHandler : facetHandlers)
{
map.put(facetHandler.getName(),facetHandler.getFieldValues(reader,doc));
rawMap.put(facetHandler.getName(),facetHandler.getRawFieldValues(reader,doc));
if (selectSet == null ||
selectSet.size() == 0 ||
selectSet.contains(facetHandler.getName()))
{
map.put(facetHandler.getName(),facetHandler.getFieldValues(reader,doc));
rawMap.put(facetHandler.getName(),facetHandler.getRawFieldValues(reader,doc));
}
}
hit.setFieldValues(map);
hit.setRawFieldValues(rawMap);
Expand Down
14 changes: 14 additions & 0 deletions sensei-core/src/main/java/com/senseidb/search/req/SenseiRequest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
Expand Down Expand Up @@ -53,6 +54,7 @@ public class SenseiRequest implements AbstractSenseiRequest, Cloneable
private int _maxPerGroup;
private Set<String> _termVectorsToFetch;
private List<String> _selectList; // Select list (mostly used in BQL)
private transient Set<String> _selectSet;
private SenseiMapReduce mapReduceFunction;
private List<SenseiError> errors;

Expand Down Expand Up @@ -432,6 +434,7 @@ public void setSort(SortField[] sorts){
public void setSelectList(List<String> selectList)
{
_selectList = selectList;
_selectSet = null;
}

/**
Expand All @@ -442,6 +445,17 @@ public List<String> getSelectList()
{
return _selectList;
}

public Set<String> getSelectSet()
{
if (_selectSet == null &&
_selectList != null &&
!(_selectList.size() == 1 && "*".equals(_selectList.get(0))))
{
_selectSet = new HashSet<String>(_selectList);
}
return _selectSet;
}

/** Represents sorting by document score (relevancy). */
public static final SortField FIELD_SCORE = new SortField (null, SortField.SCORE);
Expand Down
7 changes: 1 addition & 6 deletions sensei-core/src/main/java/com/senseidb/servlet/DefaultSenseiJSONServlet.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -329,12 +329,7 @@ public static String buildJSONResultString(SenseiRequest req, SenseiResult res)
public static JSONArray buildJSONHits(SenseiRequest req, SenseiHit[] hits)
throws Exception
{
Set<String> selectSet = null;
List<String> selectList = req.getSelectList();
if (selectList != null && !(selectList.size() == 1 && "*".equals(selectList.get(0))))
{
selectSet = new HashSet<String>(selectList);
}
Set<String> selectSet = req.getSelectSet();

JSONArray hitArray = new FastJSONArray();
for (SenseiHit hit : hits)
Expand Down
37 changes: 33 additions & 4 deletions sensei-core/src/main/java/com/senseidb/svc/impl/CoreSenseiServiceImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,9 @@

import java.io.IOException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
Expand Down Expand Up @@ -65,7 +67,7 @@ public CoreSenseiServiceImpl(SenseiCore core) {
super(core);
}

private SenseiResult browse(MultiBoboBrowser browser, BrowseRequest req, SubReaderAccessor<BoboIndexReader> subReaderAccessor) throws BrowseException
private SenseiResult browse(SenseiRequest senseiRequest, MultiBoboBrowser browser, BrowseRequest req, SubReaderAccessor<BoboIndexReader> subReaderAccessor) throws BrowseException
{
final SenseiResult result = new SenseiResult();

Expand All @@ -90,6 +92,7 @@ private SenseiResult browse(MultiBoboBrowser browser, BrowseRequest req, SubRead
result.setMapReduceResult(req.getMapReduceWrapper().getResult());
}
SenseiHit[] senseiHits = new SenseiHit[hits.length];
Set<String> selectSet = senseiRequest.getSelectSet();
for (int i = 0; i < hits.length; i++)
{
BrowseHit hit = hits[i];
Expand All @@ -104,8 +107,34 @@ private SenseiResult browse(MultiBoboBrowser browser, BrowseRequest req, SubRead
senseiHit.setDocid(docid);
senseiHit.setScore(hit.getScore());
senseiHit.setComparable(hit.getComparable());
senseiHit.setFieldValues(hit.getFieldValues());
senseiHit.setRawFieldValues(hit.getRawFieldValues());
if (selectSet != null && selectSet.size() != 0)
{
// Clear the data those are not used:
if (hit.getFieldValues() != null)
{
Iterator<String> iter = hit.getFieldValues().keySet().iterator();
while (iter.hasNext())
{
if (!selectSet.contains(iter.next()))
{
iter.remove();
}
}
}
if (hit.getRawFieldValues() != null)
{
Iterator<String> iter = hit.getRawFieldValues().keySet().iterator();
while (iter.hasNext())
{
if (!selectSet.contains(iter.next()))
{
iter.remove();
}
}
}
}
senseiHit.setFieldValues(hit.getFieldValues());
senseiHit.setRawFieldValues(hit.getRawFieldValues());
senseiHit.setStoredFields(hit.getStoredFields());
senseiHit.setExplanation(hit.getExplanation());
senseiHit.setGroupField(hit.getGroupField());
Expand Down Expand Up @@ -188,7 +217,7 @@ public List<BoboIndexReader> call() throws Exception {
}
SubReaderAccessor<BoboIndexReader> subReaderAccessor =
ZoieIndexReader.getSubReaderAccessor(validatedSegmentReaders);
SenseiResult res = browse(browser, breq, subReaderAccessor);
SenseiResult res = browse(request, browser, breq, subReaderAccessor);
int totalDocs = res.getTotalDocs()+skipDocs.get();
res.setTotalDocs(totalDocs);
return res;
Expand Down
25 changes: 21 additions & 4 deletions sensei-federated-broker/src/main/java/com/senseidb/federated/broker/FederatedBroker.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,12 @@
import com.senseidb.svc.api.SenseiException;
import com.senseidb.util.RequestConverter2;

import static com.senseidb.servlet.SenseiSearchServletParams.PARAM_RESULT_NUMHITS;
import static com.senseidb.servlet.SenseiSearchServletParams.PARAM_RESULT_TOTALDOCS;

public class FederatedBroker implements Broker<SenseiRequest, SenseiResult>{
private final static Logger logger = Logger.getLogger(SenseiBrokerProxy.class);
private final static Logger queryLogger = Logger.getLogger("com.sensei.querylog");
private List<BrokerProxy> proxies;
private int numThreads = 10;
private ExecutorService executor;
Expand Down Expand Up @@ -104,10 +108,23 @@ public void setInMemorySenseiService(InMemorySenseiService inMemorySenseiService
}
public JSONObject query(JSONObject request) {
try {
SenseiRequest senseiRequest = RequestConverter2.fromJSON(request, facetInfo);
SenseiResult senseiResult = browse(senseiRequest);
JSONObject jsonResult = DefaultSenseiJSONServlet.buildJSONResult(senseiRequest, senseiResult);
return jsonResult;
long time = System.currentTimeMillis();
int numHits = 0, totalDocs = 0;
try {
SenseiRequest senseiRequest = RequestConverter2.fromJSON(request, facetInfo);
SenseiResult senseiResult = browse(senseiRequest);
JSONObject jsonResult = DefaultSenseiJSONServlet.buildJSONResult(senseiRequest, senseiResult);
if (jsonResult != null) {
numHits = jsonResult.optInt(PARAM_RESULT_NUMHITS);
totalDocs = jsonResult.optInt(PARAM_RESULT_TOTALDOCS);
}
return jsonResult;
}
finally {
if (queryLogger.isInfoEnabled() && request != null) {
queryLogger.info(String.format("hits(%d/%d) took %dms: %s", numHits, totalDocs, System.currentTimeMillis() - time, request.toString()));
}
}
} catch (Exception ex) {
throw new RuntimeException(ex);
}
Expand Down

0 comments on commit 23218ac

Please sign in to comment.