Skip to content

Commit

Permalink
refact(data): optimize stack on high dimension data
Browse files Browse the repository at this point in the history
  • Loading branch information
@pissang
pissang committed Jul 22, 2021
1 parent 0d5886b commit 5bc0c51
Show file tree
Hide file tree
Showing 7 changed files with 123 additions and 94 deletions.
11 changes: 9 additions & 2 deletions src/chart/helper/createSeriesData.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -159,7 +159,6 @@ function createSeriesData(

const data = new SeriesData(dimInfoList, seriesModel);
data.setCalculationInfo(stackCalculationInfo);
console.log(stackCalculationInfo)

const dimValueGetter =
firstCategoryDimIndex != null
Expand All @@ -171,11 +170,19 @@ function createSeriesData(
: this.defaultDimValueGetter(itemOpt, dimName, dataIndex, dimIndex);
}
: null;
let storage;
if (!isOriginalSource) {
storage = sourceManager.getSharedDataStorage(dimInfoList);
if (stackCalculationInfo.stackedOverDimension) {
storage.appendDimension(stackCalculationInfo.stackedOverDimension, 'float');
storage.appendDimension(stackCalculationInfo.stackResultDimension, 'float');
}
}

data.hasItemOption = false;
data.initData(
// Try to reuse the data storage in sourceManager if using dataset.
isOriginalSource ? source : sourceManager.getDataStorage(dimInfoList)
isOriginalSource ? source : storage
, null, dimValueGetter);

return data;
Expand Down
60 changes: 47 additions & 13 deletions src/data/DataStorage.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -200,28 +200,48 @@ class DataStorage {
}) as DataStorageDimensionDefine);

const dimensionsIdxMap: Dictionary<number> = {};
let prefix = '';
let needsHasOwn = false;

// Needs to add prefix if key is used in object prototype
for (let i = 0; i < dimensions.length; i++) {
const name = dimensions[i].name;
if ((emptyObj as any)[name] != null) {
prefix = '$$';
needsHasOwn = true;
break;
}
}
for (let i = 0; i < dimensions.length; i++) {
const dim = dimensions[i];
const name = dim.name;
dimensionsIdxMap[prefix + name] = i;
dimensionsIdxMap[name] = i;
}

// We use different functions because it may be a hotspot code.
this.getDimensionIndex = prefix ? function (dim) {
return dimensionsIdxMap[prefix + dim];
} : function (dim) {
return dimensionsIdxMap[dim];
const updateGetDimensionIndex = () => {
this.getDimensionIndex = needsHasOwn ? function (dim) {
return dimensionsIdxMap.hasOwnProperty(dim) ? dimensionsIdxMap[dim] : undefined;
} : function (dim) {
return dimensionsIdxMap[dim];
};
};

this.appendDimension = (dimName, dimType) => {
if (!needsHasOwn && (emptyObj as any)[dimName] != null) {
needsHasOwn = true;
updateGetDimensionIndex();
}
const dimensions = this._dimensions;
const idx = dimensions.length;
dimensions.push({
name: dimName,
type: dimType
});
this._chunks.push(new dataCtors[dimType || 'float'](this._rawCount));
this._rawExtent.push(getInitialExtent());
dimensionsIdxMap[dimName] = idx;
};

updateGetDimensionIndex();

this._initDataFromProvider(0, provider.count());
}

Expand All @@ -234,6 +254,7 @@ class DataStorage {
}

getDimensionIndex: (dim: DimensionName) => number;
appendDimension: (dim: DimensionName, type: DataStorageDimensionType) => void;

getDimensionCount() {
return this._dimensions.length;
Expand Down Expand Up @@ -833,12 +854,27 @@ class DataStorage {
map(dims: DimensionIndex[], cb: MapCb): DataStorage {
// TODO only clone picked chunks.
const target = this.clone(dims);
const targetChunks = target._chunks;
this._updateDims(target, dims, cb);
return target;
}

/**
* Danger only can be used in SeriesData.
*/
modify(dims: DimensionIndex[], cb: MapCb) {
this._updateDims(this, dims, cb);
}

_updateDims(
target: DataStorage,
dims: DimensionIndex[],
cb: MapCb
) {
const targetChunks = target._chunks;

const tmpRetValue = [];
const dimSize = dims.length;
const dataCount = this.count();
const dataCount = target.count();
const values = [];
const rawExtent = target._rawExtent;

Expand All @@ -847,7 +883,7 @@ class DataStorage {
}

for (let dataIndex = 0; dataIndex < dataCount; dataIndex++) {
const rawIndex = this.getRawIndex(dataIndex);
const rawIndex = target.getRawIndex(dataIndex);

for (let k = 0; k < dimSize; k++) {
values[k] = targetChunks[dims[k]][rawIndex];
Expand Down Expand Up @@ -881,8 +917,6 @@ class DataStorage {
}
}
}

return target;
}

/**
Expand Down
97 changes: 53 additions & 44 deletions src/data/SeriesData.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -321,7 +321,13 @@ class SeriesData<
}

private _getStoreDimIndex(dim: DimensionLoose): DimensionIndex {
return this._store.getDimensionIndex(this.getDimension(dim));
const dimIdx = this._store.getDimensionIndex(this.getDimension(dim));
if (__DEV__) {
if (dimIdx == null) {
throw new Error('Unkown dimension ' + dim);
}
}
return dimIdx;
}

/**
Expand Down Expand Up @@ -389,14 +395,7 @@ class SeriesData<
const dimensions = this.dimensions;
const dimensionInfos = map(dimensions, dimName => this._dimensionInfos[dimName]);
if (data instanceof DataStorage) {
if (data.canUse(dimensionInfos)) {
store = data;
}
// Sync failed
else {
// Needs to recreate data storage if it's not match the given dimension.
data = data.getSource();
}
store = data;
}

if (!store) {
Expand Down Expand Up @@ -764,10 +763,6 @@ class SeriesData<

const dimIndices = map(normalizeDimensions(dims), this._getStoreDimIndex, this);

if (__DEV__) {
validateDimensions(this, dimIndices);
}

this._store.each(dimIndices, (fCtx
? zrUtil.bind(cb as any, fCtx as any)
: cb) as any
Expand Down Expand Up @@ -799,10 +794,6 @@ class SeriesData<

const dimIndices = map(normalizeDimensions(dims), this._getStoreDimIndex, this);

if (__DEV__) {
validateDimensions(this, dimIndices);
}

// Clone first
this._store = this._store.clone();
this._store.filterSelf(dimIndices, (fCtx
Expand All @@ -829,10 +820,6 @@ class SeriesData<
dimIndices.push(dimIdx);
});

if (__DEV__) {
validateDimensions(this, dimIndices);
}

this._store = this._store.clone();
this._store.selectRange(innerRange);
return this;
Expand Down Expand Up @@ -893,10 +880,6 @@ class SeriesData<
normalizeDimensions(dims), this._getStoreDimIndex, this
);

if (__DEV__) {
validateDimensions(this, dimIndices);
}

const list = cloneListForMapAndSample(this);
list._store = this._store.map(dimIndices, (fCtx
? zrUtil.bind(cb as any, fCtx as any)
Expand All @@ -905,6 +888,41 @@ class SeriesData<
return list;
}

/**
* !!Danger: used on stack dimension only.
*/
modify<Ctx>(dims: DimensionLoose, cb: MapCb1<Ctx>, ctx?: Ctx, ctxCompat?: Ctx): void;
modify<Ctx>(dims: [DimensionLoose], cb: MapCb1<Ctx>, ctx?: Ctx, ctxCompat?: Ctx): void;
/* eslint-disable-next-line */
modify<Ctx>(dims: [DimensionLoose, DimensionLoose], cb: MapCb2<Ctx>, ctx?: Ctx, ctxCompat?: Ctx): void;
modify<Ctx>(
dims: ItrParamDims,
cb: MapCb<Ctx>,
ctx?: Ctx,
ctxCompat?: Ctx
) {
// ctxCompat just for compat echarts3
const fCtx = (ctx || ctxCompat || this) as CtxOrList<Ctx>;

if (__DEV__) {
zrUtil.each(normalizeDimensions(dims), dim => {
const dimInfo = this.getDimensionInfo(dim);
if (!dimInfo.isCalculationCoord) {
console.error('Danger: only stack dimension can be modified');
}
});
}

const dimIndices = map(
normalizeDimensions(dims), this._getStoreDimIndex, this
);

this._store.modify(dimIndices, (fCtx
? zrUtil.bind(cb as any, fCtx as any)
: cb) as any
);
}

/**
* Large data down sampling on given dimension
* @param sampleIndex Sample index for name and id
Expand Down Expand Up @@ -1207,13 +1225,13 @@ class SeriesData<
// ----------------------------------------------------------
private static internalField = (function () {

prepareInvertedIndex = function (list: SeriesData): void {
const invertedIndicesMap = list._invertedIndicesMap;
prepareInvertedIndex = function (data: SeriesData): void {
const invertedIndicesMap = data._invertedIndicesMap;
zrUtil.each(invertedIndicesMap, function (invertedIndices, dim) {
const dimInfo = list._dimensionInfos[dim];
const dimInfo = data._dimensionInfos[dim];
// Currently, only dimensions that has ordinalMeta can create inverted indices.
const ordinalMeta = dimInfo.ordinalMeta;
const store = list._store;
const store = data._store;
const dimIdx = store.getDimensionIndex(dim);
if (ordinalMeta) {
invertedIndices = invertedIndicesMap[dim] = new CtorInt32Array(
Expand All @@ -1233,18 +1251,18 @@ class SeriesData<
};

getIdNameFromStore = function (
list: SeriesData, dimIdx: number, idx: number
data: SeriesData, dimIdx: number, idx: number
): string {
return convertOptionIdName(list._getCategory(dimIdx, idx), null);
return convertOptionIdName(data._getCategory(dimIdx, idx), null);
};

/**
* @see the comment of `List['getId']`.
*/
getId = function (list: SeriesData, rawIndex: number): string {
let id = list._idList[rawIndex];
if (id == null && list._idDimIdx != null) {
id = getIdNameFromStore(list, list._idDimIdx, rawIndex);
getId = function (data: SeriesData, rawIndex: number): string {
let id = data._idList[rawIndex];
if (id == null && data._idDimIdx != null) {
id = getIdNameFromStore(data, data._idDimIdx, rawIndex);
}
if (id == null) {
id = ID_PREFIX + rawIndex;
Expand All @@ -1261,15 +1279,6 @@ class SeriesData<
return dimensions;
};

validateDimensions = function (list: SeriesData, dims: DimensionIndex[]): void {
for (let i = 0; i < dims.length; i++) {
// stroage may be empty when no data, so use
// dimensionInfos to check.
if (!list.dimensions[dims[i]]) {
console.error('Unkown dimension ' + dims[i]);
}
}
};

// Data in excludeDimensions is copied, otherwise transfered.
cloneListForMapAndSample = function (original: SeriesData): SeriesData {
Expand Down
6 changes: 4 additions & 2 deletions src/data/helper/dataStackHelper.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,8 +104,10 @@ export function enableDataStack(
// might not be a good way.
if (stackedDimInfo) {
// Use a weird name that not duplicated with other names.
stackResultDimension = '__\0ecstackresult';
stackedOverDimension = '__\0ecstackedover';
// Also need to use seriesModel.id as postfix because different
// series may share same data storage. The stack dimension needs to be distinguished.
stackResultDimension = '__\0ecstackresult_' + seriesModel.id;
stackedOverDimension = '__\0ecstackedover_' + seriesModel.id;

// Create inverted index to fast query index by value.
if (stackedByDimInfo) {
Expand Down
4 changes: 3 additions & 1 deletion src/data/helper/sourceManager.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -364,11 +364,13 @@ export class SourceManager {
}

/**
*
* Get a data storage which can be shared across series.
* Only available for series.
*
* @param dimensions Dimensions that are generated in series.
*/
getDataStorage(seriesDims: SeriesDimensionDefine[]): DataStorage {
getSharedDataStorage(seriesDims: SeriesDimensionDefine[]): DataStorage {
if (__DEV__) {
assert(isSeries(this._sourceHost), 'Can only call getDataStorage on series source manager.');
}
Expand Down
6 changes: 1 addition & 5 deletions src/processor/dataStack.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ function calculateStack(stackInfoList: StackInfo[]) {

// Should not write on raw data, because stack series model list changes
// depending on legend selection.
const newData = targetData.map(dims, function (v0, v1, dataIndex) {
targetData.modify(dims, function (v0, v1, dataIndex) {
let sum = targetData.get(targetStackInfo.stackedDimension, dataIndex) as number;

// Consider `connectNulls` of line area, if value is NaN, stackedOver
Expand Down Expand Up @@ -141,9 +141,5 @@ function calculateStack(stackInfoList: StackInfo[]) {

return resultVal;
});

(targetData.hostModel as SeriesModel).setData(newData);
// Update for consequent calculation
targetStackInfo.data = newData;
});
}
Loading

0 comments on commit 5bc0c51

Please sign in to comment.