forked from apache/jackrabbit-oak
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
OAK-8788: Add index creation and index completion time to indexes
git-svn-id: https://svn.apache.org/repos/asf/jackrabbit/oak/trunk@1873532 13f79535-47bb-0310-9956-ffa450edef68
- Loading branch information
Showing
10 changed files
with
891 additions
and
2 deletions.
There are no files selected for viewing
189 changes: 189 additions & 0 deletions
189
oak-run/src/main/java/org/apache/jackrabbit/oak/indexversion/IndexVersionOperation.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,189 @@ | ||
| /* | ||
| * Licensed to the Apache Software Foundation (ASF) under one | ||
| * or more contributor license agreements. See the NOTICE file | ||
| * distributed with this work for additional information | ||
| * regarding copyright ownership. The ASF licenses this file | ||
| * to you under the Apache License, Version 2.0 (the | ||
| * "License"); you may not use this file except in compliance | ||
| * with the License. You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, | ||
| * software distributed under the License is distributed on an | ||
| * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | ||
| * KIND, either express or implied. See the License for the | ||
| * specific language governing permissions and limitations | ||
| * under the License. | ||
| */ | ||
| package org.apache.jackrabbit.oak.indexversion; | ||
|
|
||
| import org.apache.jackrabbit.oak.api.Type; | ||
| import org.apache.jackrabbit.oak.commons.PathUtils; | ||
| import org.apache.jackrabbit.oak.plugins.index.search.IndexDefinition; | ||
| import org.apache.jackrabbit.oak.plugins.index.search.spi.query.IndexName; | ||
| import org.apache.jackrabbit.oak.spi.state.NodeState; | ||
| import org.slf4j.Logger; | ||
| import org.slf4j.LoggerFactory; | ||
|
|
||
| import java.util.ArrayList; | ||
| import java.util.Collections; | ||
| import java.util.LinkedList; | ||
| import java.util.List; | ||
|
|
||
| /* | ||
| Main operation of this class is to mark IndexName's with operations | ||
| */ | ||
| public class IndexVersionOperation { | ||
| private static final Logger LOG = LoggerFactory.getLogger(IndexVersionOperation.class); | ||
|
|
||
| private IndexName indexName; | ||
| private Operation operation; | ||
|
|
||
| public IndexVersionOperation(IndexName indexName) { | ||
| this.indexName = indexName; | ||
| this.operation = Operation.NOOP; | ||
| } | ||
|
|
||
| public void setOperation(Operation operation) { | ||
| this.operation = operation; | ||
| } | ||
|
|
||
| public Operation getOperation() { | ||
| return this.operation; | ||
| } | ||
|
|
||
| public IndexName getIndexName() { | ||
| return this.indexName; | ||
| } | ||
|
|
||
| @Override | ||
| public String toString() { | ||
| return this.getIndexName() + " operation:" + this.getOperation(); | ||
| } | ||
|
|
||
| /** | ||
| * @param indexDefParentNode NodeState of parent of baseIndex | ||
| * @param indexNameObjectList This is a list of IndexName Objects with same baseIndexName on which operations will be applied. | ||
| * @param purgeThresholdMillis after which a fully functional index is eligible for purge operations | ||
| * @return This method returns an IndexVersionOperation list i.e indexNameObjectList marked with operations | ||
| */ | ||
| public static List<IndexVersionOperation> generateIndexVersionOperationList(NodeState indexDefParentNode, | ||
| List<IndexName> indexNameObjectList, long purgeThresholdMillis) { | ||
| int activeProductVersion = -1; | ||
| List<IndexName> reverseSortedIndexNameList = getReverseSortedIndexNameList(indexNameObjectList); | ||
| removeDisabledCustomIndexesFromList(indexDefParentNode, reverseSortedIndexNameList); | ||
| List<IndexVersionOperation> indexVersionOperationList = new LinkedList<>(); | ||
| for (IndexName indexNameObject : reverseSortedIndexNameList) { | ||
| NodeState indexNode = indexDefParentNode | ||
| .getChildNode(PathUtils.getName(indexNameObject.getNodeName())); | ||
| /* | ||
| All Indexes are by default NOOP until we get index which is active for more than threshold limit | ||
| */ | ||
| if (activeProductVersion == -1) { | ||
| indexVersionOperationList.add(new IndexVersionOperation(indexNameObject)); | ||
| if (indexNode.hasChildNode(IndexDefinition.STATUS_NODE)) { | ||
| if (indexNode.getChildNode(IndexDefinition.STATUS_NODE) | ||
| .getProperty(IndexDefinition.REINDEX_COMPLETION_TIMESTAMP) != null) { | ||
| String reindexCompletionTime = indexDefParentNode | ||
| .getChildNode(PathUtils.getName(indexNameObject.getNodeName())) | ||
| .getChildNode(IndexDefinition.STATUS_NODE) | ||
| .getProperty(IndexDefinition.REINDEX_COMPLETION_TIMESTAMP).getValue(Type.DATE); | ||
| long reindexCompletionTimeInMillis = PurgeOldVersionUtils.getMillisFromString(reindexCompletionTime); | ||
| long currentTimeInMillis = System.currentTimeMillis(); | ||
| if (currentTimeInMillis - reindexCompletionTimeInMillis > purgeThresholdMillis) { | ||
| activeProductVersion = indexNameObject.getProductVersion(); | ||
| } | ||
| } else { | ||
| LOG.warn(IndexDefinition.REINDEX_COMPLETION_TIMESTAMP | ||
| + " property is not set for index " + indexNameObject.getNodeName()); | ||
| } | ||
| } | ||
| } else { | ||
| /* | ||
| Once we found active index, we mark all its custom version with DELETE | ||
| and OOTB same product version index with DELETE_HIDDEN_AND_DISABLE | ||
| */ | ||
| if (indexNameObject.getProductVersion() == activeProductVersion | ||
| && indexNameObject.getCustomerVersion() == 0) { | ||
| IndexVersionOperation indexVersionOperation = new IndexVersionOperation(indexNameObject); | ||
| indexVersionOperation.setOperation(Operation.DELETE_HIDDEN_AND_DISABLE); | ||
| indexVersionOperationList.add(indexVersionOperation); | ||
| } else { | ||
| IndexVersionOperation indexVersionOperation = new IndexVersionOperation(indexNameObject); | ||
| indexVersionOperation.setOperation(Operation.DELETE); | ||
| indexVersionOperationList.add(indexVersionOperation); | ||
| } | ||
| } | ||
| } | ||
| if (!isValidIndexVersionOperationList(indexVersionOperationList)) { | ||
| indexVersionOperationList = Collections.emptyList(); | ||
| } | ||
| return indexVersionOperationList; | ||
| } | ||
|
|
||
| /* | ||
| returns IndexNameObjects in descending order of version i.e from newest version to oldest | ||
| */ | ||
| private static List<IndexName> getReverseSortedIndexNameList(List<IndexName> indexNameObjectList) { | ||
| List<IndexName> reverseSortedIndexNameObjectList = new ArrayList<>(indexNameObjectList); | ||
| Collections.sort(reverseSortedIndexNameObjectList, Collections.reverseOrder()); | ||
| return reverseSortedIndexNameObjectList; | ||
| } | ||
|
|
||
| /** | ||
| * @param indexVersionOperations | ||
| * @return true if the IndexVersionOperation list passes following criteria. | ||
| * For merging indexes we need baseIndex and latest custom index. | ||
| * So we first validate that if there are custom indexes than OOTB index with same product must be marked with DELETE_HIDDEN_AND_DISABLE | ||
| */ | ||
| private static boolean isValidIndexVersionOperationList(List<IndexVersionOperation> indexVersionOperations) { | ||
| boolean isValid = false; | ||
| IndexVersionOperation lastNoopOperationIndexVersion = null; | ||
| IndexVersionOperation indexWithDeleteHiddenOp = null; | ||
| for (IndexVersionOperation indexVersionOperation : indexVersionOperations) { | ||
| if (indexVersionOperation.getOperation() == Operation.NOOP) { | ||
| lastNoopOperationIndexVersion = indexVersionOperation; | ||
| } | ||
| if (indexVersionOperation.getOperation() == Operation.DELETE_HIDDEN_AND_DISABLE) { | ||
| indexWithDeleteHiddenOp = indexVersionOperation; | ||
| } | ||
| } | ||
| if (lastNoopOperationIndexVersion.getIndexName().getCustomerVersion() == 0) { | ||
| isValid = true; | ||
| } else if (lastNoopOperationIndexVersion.getIndexName().getCustomerVersion() != 0) { | ||
| if (indexWithDeleteHiddenOp != null | ||
| && lastNoopOperationIndexVersion.getIndexName().getProductVersion() == indexWithDeleteHiddenOp.getIndexName().getProductVersion()) { | ||
| isValid = true; | ||
| } | ||
| } | ||
| if (!isValid) { | ||
| LOG.info("IndexVersionOperation List is not valid for index {}", lastNoopOperationIndexVersion.getIndexName().getNodeName()); | ||
| } | ||
| return isValid; | ||
| } | ||
|
|
||
| private static void removeDisabledCustomIndexesFromList(NodeState indexDefParentNode, | ||
| List<IndexName> indexNameObjectList) { | ||
| for (int i = 0; i < indexNameObjectList.size(); i++) { | ||
| NodeState indexNode = indexDefParentNode | ||
| .getChildNode(PathUtils.getName(indexNameObjectList.get(i).getNodeName())); | ||
| if (indexNode.getProperty("type") != null && "disabled".equals(indexNode.getProperty("type").getValue(Type.STRING))) { | ||
| indexNameObjectList.remove(i); | ||
| } | ||
| } | ||
| } | ||
|
|
||
| /* | ||
| NOOP means : No operation to be performed index Node | ||
| DELETE_HIDDEN_AND_DISABLE: This operation means that we should disable this indexNode and delete all hidden nodes under it | ||
| DELETE: Delete this index altogether | ||
| */ | ||
| enum Operation { | ||
| NOOP, | ||
| DELETE_HIDDEN_AND_DISABLE, | ||
| DELETE | ||
| } | ||
| } | ||
|
|
||
|
|
170 changes: 170 additions & 0 deletions
170
oak-run/src/main/java/org/apache/jackrabbit/oak/indexversion/PurgeOldIndexVersion.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,170 @@ | ||
| /* | ||
| * Licensed to the Apache Software Foundation (ASF) under one | ||
| * or more contributor license agreements. See the NOTICE file | ||
| * distributed with this work for additional information | ||
| * regarding copyright ownership. The ASF licenses this file | ||
| * to you under the Apache License, Version 2.0 (the | ||
| * "License"); you may not use this file except in compliance | ||
| * with the License. You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, | ||
| * software distributed under the License is distributed on an | ||
| * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | ||
| * KIND, either express or implied. See the License for the | ||
| * specific language governing permissions and limitations | ||
| * under the License. | ||
| */ | ||
| package org.apache.jackrabbit.oak.indexversion; | ||
|
|
||
| import org.apache.jackrabbit.oak.api.CommitFailedException; | ||
| import org.apache.jackrabbit.oak.api.Type; | ||
| import org.apache.jackrabbit.oak.commons.PathUtils; | ||
| import org.apache.jackrabbit.oak.plugins.index.IndexPathService; | ||
| import org.apache.jackrabbit.oak.plugins.index.IndexPathServiceImpl; | ||
| import org.apache.jackrabbit.oak.plugins.index.IndexUpdateProvider; | ||
| import org.apache.jackrabbit.oak.plugins.index.property.PropertyIndexEditorProvider; | ||
| import org.apache.jackrabbit.oak.plugins.index.search.spi.query.IndexName; | ||
| import org.apache.jackrabbit.oak.run.cli.NodeStoreFixture; | ||
| import org.apache.jackrabbit.oak.run.cli.NodeStoreFixtureProvider; | ||
| import org.apache.jackrabbit.oak.run.cli.Options; | ||
| import org.apache.jackrabbit.oak.spi.commit.CommitInfo; | ||
| import org.apache.jackrabbit.oak.spi.commit.EditorHook; | ||
| import org.apache.jackrabbit.oak.spi.state.NodeBuilder; | ||
| import org.apache.jackrabbit.oak.spi.state.NodeState; | ||
| import org.apache.jackrabbit.oak.spi.state.NodeStateUtils; | ||
| import org.apache.jackrabbit.oak.spi.state.NodeStore; | ||
| import org.slf4j.Logger; | ||
| import org.slf4j.LoggerFactory; | ||
|
|
||
| import java.io.IOException; | ||
| import java.util.ArrayList; | ||
| import java.util.HashMap; | ||
| import java.util.HashSet; | ||
| import java.util.List; | ||
| import java.util.Map; | ||
| import java.util.Set; | ||
|
|
||
| public class PurgeOldIndexVersion { | ||
| private static final Logger LOG = LoggerFactory.getLogger(PurgeOldIndexVersion.class); | ||
|
|
||
| public void execute(Options opts, long purgeThresholdMillis, List<String> indexPaths) throws Exception { | ||
| boolean isReadWriteRepository = opts.getCommonOpts().isReadWrite(); | ||
| if (!isReadWriteRepository) { | ||
| LOG.info("Repository connected in read-only mode. Use '--read-write' for write operations"); | ||
| } | ||
| try (NodeStoreFixture fixture = NodeStoreFixtureProvider.create(opts)) { | ||
| NodeStore nodeStore = fixture.getStore(); | ||
| List<String> sanitisedIndexPaths = sanitiseUserIndexPaths(indexPaths); | ||
| Set<String> indexPathSet = filterIndexPaths(getRepositoryIndexPaths(nodeStore), sanitisedIndexPaths); | ||
| Map<String, Set<String>> segregateIndexes = segregateIndexes(indexPathSet); | ||
| for (Map.Entry<String, Set<String>> entry : segregateIndexes.entrySet()) { | ||
| String baseIndexPath = entry.getKey(); | ||
| String parentPath = PathUtils.getParentPath(entry.getKey()); | ||
| List<IndexName> indexNameObjectList = getIndexNameObjectList(entry.getValue()); | ||
| NodeState indexDefParentNode = NodeStateUtils.getNode(nodeStore.getRoot(), | ||
| parentPath); | ||
| List<IndexVersionOperation> toDeleteIndexNameObjectList = | ||
| IndexVersionOperation.generateIndexVersionOperationList(indexDefParentNode, indexNameObjectList, purgeThresholdMillis); | ||
| if (isReadWriteRepository && !toDeleteIndexNameObjectList.isEmpty()) { | ||
| purgeOldIndexVersion(nodeStore, toDeleteIndexNameObjectList); | ||
| } else { | ||
| LOG.info("Repository is opened in read-only mode: IndexOperations" + | ||
| " for index at path {} are : {}", baseIndexPath, toDeleteIndexNameObjectList); | ||
| } | ||
| } | ||
| } | ||
| } | ||
|
|
||
| /** | ||
| * @param userIndexPaths indexpaths provided by user | ||
| * @return a list of Indexpaths having baseIndexpaths or path till oak:index | ||
| * @throws IllegalArgumentException if the paths provided are not till oak:index or till index | ||
| */ | ||
| private List<String> sanitiseUserIndexPaths(List<String> userIndexPaths) { | ||
| List<String> sanitisedUserIndexPaths = new ArrayList<>(); | ||
| for (String userIndexPath : userIndexPaths) { | ||
| if (PathUtils.getName(userIndexPath).equals(PurgeOldVersionUtils.OAK_INDEX)) { | ||
| sanitisedUserIndexPaths.add(userIndexPath); | ||
| } else if (PathUtils.getName(PathUtils.getParentPath(userIndexPath)).equals(PurgeOldVersionUtils.OAK_INDEX)) { | ||
| sanitisedUserIndexPaths.add(IndexName.parse(userIndexPath).getBaseName()); | ||
| } else { | ||
| throw new IllegalArgumentException(userIndexPath + " indexpath is not valid"); | ||
| } | ||
| } | ||
| return sanitisedUserIndexPaths; | ||
| } | ||
|
|
||
| /** | ||
| * @param indexPathSet | ||
| * @return a map with baseIndexName as key and a set of indexpaths having same baseIndexName | ||
| */ | ||
| private Map<String, Set<String>> segregateIndexes(Set<String> indexPathSet) { | ||
| Map<String, Set<String>> segregatedIndexes = new HashMap<>(); | ||
| for (String path : indexPathSet) { | ||
| String baseIndexPath = IndexName.parse(path).getBaseName(); | ||
| Set<String> indexPaths = segregatedIndexes.get(baseIndexPath); | ||
| if (indexPaths == null) { | ||
| indexPaths = new HashSet<>(); | ||
| } | ||
| indexPaths.add(path); | ||
| segregatedIndexes.put(baseIndexPath, indexPaths); | ||
| } | ||
| return segregatedIndexes; | ||
| } | ||
|
|
||
| private Iterable<String> getRepositoryIndexPaths(NodeStore store) throws CommitFailedException, IOException { | ||
| IndexPathService indexPathService = new IndexPathServiceImpl(store); | ||
| return indexPathService.getIndexPaths(); | ||
| } | ||
|
|
||
|
|
||
| /** | ||
| * @param repositoryIndexPaths: list of indexpaths retrieved from index service | ||
| * @param commandlineIndexPaths indexpaths provided by user | ||
| * @return returns set of indexpaths which are to be considered for purging | ||
| */ | ||
| private Set<String> filterIndexPaths(Iterable<String> repositoryIndexPaths, List<String> commandlineIndexPaths) { | ||
| Set<String> filteredIndexPaths = new HashSet<>(); | ||
| for (String commandlineIndexPath : commandlineIndexPaths) { | ||
| for (String repositoryIndexPath : repositoryIndexPaths) { | ||
| if (PurgeOldVersionUtils.isIndexChildNode(commandlineIndexPath, repositoryIndexPath) | ||
| || PurgeOldVersionUtils.isBaseIndexEqual(commandlineIndexPath, repositoryIndexPath)) { | ||
| filteredIndexPaths.add(repositoryIndexPath); | ||
| } | ||
| } | ||
| } | ||
| return filteredIndexPaths; | ||
| } | ||
|
|
||
| private List<IndexName> getIndexNameObjectList(Set<String> versionedIndexPaths) { | ||
| List<IndexName> indexNameObjectList = new ArrayList<>(); | ||
| for (String indexNameString : versionedIndexPaths) { | ||
| indexNameObjectList.add(IndexName.parse(indexNameString)); | ||
| } | ||
| return indexNameObjectList; | ||
| } | ||
|
|
||
| private void purgeOldIndexVersion(NodeStore store, | ||
| List<IndexVersionOperation> toDeleteIndexNameObjectList) throws CommitFailedException { | ||
| for (IndexVersionOperation toDeleteIndexNameObject : toDeleteIndexNameObjectList) { | ||
| NodeState root = store.getRoot(); | ||
| NodeBuilder rootBuilder = root.builder(); | ||
| NodeBuilder nodeBuilder = PurgeOldVersionUtils.getNode(rootBuilder, toDeleteIndexNameObject.getIndexName().getNodeName()); | ||
| if (nodeBuilder.exists()) { | ||
| if (toDeleteIndexNameObject.getOperation() == IndexVersionOperation.Operation.DELETE_HIDDEN_AND_DISABLE) { | ||
| nodeBuilder.setProperty("type", "disabled", Type.STRING); | ||
| PurgeOldVersionUtils.recursiveDeleteHiddenChildNodes(store, toDeleteIndexNameObject.getIndexName().getNodeName()); | ||
| } else if (toDeleteIndexNameObject.getOperation() == IndexVersionOperation.Operation.DELETE) { | ||
| nodeBuilder.remove(); | ||
| } | ||
| EditorHook hook = new EditorHook( | ||
| new IndexUpdateProvider(new PropertyIndexEditorProvider())); | ||
| store.merge(rootBuilder, hook, CommitInfo.EMPTY); | ||
| } else { | ||
| LOG.error("nodebuilder null for path " + toDeleteIndexNameObject.getIndexName().getNodeName()); | ||
| } | ||
| } | ||
| } | ||
| } |
Oops, something went wrong.