forked from oracle/opengrok
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
detect repositories for each source root subdirectory in parallel
fixes oracle#773
- Loading branch information
Vladimir Kotal
committed
Oct 6, 2020
1 parent
d5999cd
commit 8df660a
Showing
6 changed files
with
143 additions
and
19 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -18,7 +18,7 @@ | |
| */ | ||
|
|
||
| /* | ||
| * Copyright (c) 2005, 2019, Oracle and/or its affiliates. All rights reserved. | ||
| * Copyright (c) 2005, 2020, Oracle and/or its affiliates. All rights reserved. | ||
| * Portions Copyright (c) 2017-2020, Chris Fraire <[email protected]>. | ||
| */ | ||
| package org.opengrok.indexer.history; | ||
|
|
@@ -41,6 +41,7 @@ | |
| import java.util.concurrent.CountDownLatch; | ||
| import java.util.concurrent.ExecutorService; | ||
| import java.util.concurrent.Executors; | ||
| import java.util.concurrent.Future; | ||
| import java.util.concurrent.ThreadFactory; | ||
| import java.util.logging.Level; | ||
| import java.util.logging.Logger; | ||
|
|
@@ -88,15 +89,12 @@ public final class HistoryGuru { | |
| */ | ||
| private final Map<String, String> repositoryRoots = new ConcurrentHashMap<>(); | ||
|
|
||
| private final int scanningDepth; | ||
|
|
||
| /** | ||
| * Creates a new instance of HistoryGuru, and try to set the default source | ||
| * control system. | ||
| */ | ||
| private HistoryGuru() { | ||
| env = RuntimeEnvironment.getInstance(); | ||
| scanningDepth = env.getScanningDepth(); | ||
|
|
||
| HistoryCache cache = null; | ||
| if (env.useHistoryCache()) { | ||
|
|
@@ -435,7 +433,7 @@ private Collection<RepositoryInfo> addRepositories(File[] files, | |
| "Failed to get sub directories for ''{0}'', " + | ||
| "check access permissions.", | ||
| file.getAbsolutePath()); | ||
| } else if (depth <= scanningDepth) { | ||
| } else if (depth <= env.getScanningDepth()) { | ||
| repoList.addAll(addRepositories(subFiles, | ||
| allowedNesting, depth + 1, isNested)); | ||
| } | ||
|
|
@@ -453,7 +451,7 @@ private Collection<RepositoryInfo> addRepositories(File[] files, | |
| LOGGER.log(Level.WARNING, | ||
| "Failed to get sub directories for ''{0}'', check access permissions.", | ||
| file.getAbsolutePath()); | ||
| } else if (depth <= scanningDepth) { | ||
| } else if (depth <= env.getScanningDepth()) { | ||
| // Search down to a limit -- if not: too much | ||
| // stat'ing for huge Mercurial repositories | ||
| repoList.addAll(addRepositories(subFiles, | ||
|
|
@@ -480,7 +478,25 @@ private Collection<RepositoryInfo> addRepositories(File[] files, | |
| * @return collection of added repositories | ||
| */ | ||
| public Collection<RepositoryInfo> addRepositories(File[] files) { | ||
| return addRepositories(files, env.getNestingMaximum(), 0, false); | ||
| ExecutorService executor = env.getIndexerParallelizer().getRepositorySearchExecutor(); | ||
| List<Future<Collection<RepositoryInfo>>> futures = new ArrayList<>(); | ||
| for (File file: files) { | ||
| futures.add(executor.submit(() -> addRepositories(new File[]{file}, | ||
| env.getNestingMaximum(), 0, false))); | ||
| } | ||
|
|
||
| List<RepositoryInfo> repoList = new ArrayList<>(); | ||
| futures.forEach(future -> { | ||
| try { | ||
| repoList.addAll(future.get()); | ||
| } catch (Exception e) { | ||
| LOGGER.log(Level.WARNING, "failed to get results of repository scan"); | ||
| } | ||
| }); | ||
|
|
||
| env.getIndexerParallelizer().bounceRepositorySearchExecutor(); | ||
|
|
||
| return repoList; | ||
| } | ||
|
|
||
| /** | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -19,6 +19,7 @@ | |
|
|
||
| /* | ||
| * Copyright (c) 2017-2020, Chris Fraire <[email protected]>. | ||
| * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. | ||
| */ | ||
|
|
||
| package org.opengrok.indexer.index; | ||
|
|
@@ -61,6 +62,7 @@ public class IndexerParallelizer implements AutoCloseable { | |
| private LazilyInstantiate<ExecutorService> lzHistoryExecutor; | ||
| private LazilyInstantiate<ExecutorService> lzHistoryRenamedExecutor; | ||
| private LazilyInstantiate<ExecutorService> lzCtagsWatcherExecutor; | ||
| private LazilyInstantiate<ExecutorService> lzRepositorySearchExecutor; | ||
|
|
||
| /** | ||
| * Initializes a new instance using settings from the specified environment | ||
|
|
@@ -84,6 +86,7 @@ public IndexerParallelizer(RuntimeEnvironment env) { | |
| createLazyHistoryExecutor(); | ||
| createLazyHistoryRenamedExecutor(); | ||
| createLazyCtagsWatcherExecutor(); | ||
| createLazyRepositorySearchExecutor(); | ||
| } | ||
|
|
||
| /** | ||
|
|
@@ -128,6 +131,13 @@ public ExecutorService getCtagsWatcherExecutor() { | |
| return lzCtagsWatcherExecutor.get(); | ||
| } | ||
|
|
||
| /** | ||
| * @return the ExecutorService used for repository scan | ||
| */ | ||
| public ExecutorService getRepositorySearchExecutor() { | ||
| return lzRepositorySearchExecutor.get(); | ||
| } | ||
|
|
||
| /** | ||
| * Calls {@link #bounce()}, which prepares for -- but does not start -- new | ||
| * pools. | ||
|
|
@@ -154,43 +164,75 @@ public void close() { | |
| * call this method satisfactorily too. | ||
| */ | ||
| public void bounce() { | ||
| bounceForkJoinPool(); | ||
| bounceFixedExecutor(); | ||
| bounceCtagsPool(); | ||
| bounceHistoryExecutor(); | ||
| bounceHistoryRenamedExecutor(); | ||
| bounceCtagsWatcherExecutor(); | ||
| bounceRepositorySearchExecutor(); | ||
| } | ||
|
|
||
| private void bounceForkJoinPool() { | ||
| if (lzForkJoinPool.isActive()) { | ||
| ForkJoinPool formerForkJoinPool = lzForkJoinPool.get(); | ||
| createLazyForkJoinPool(); | ||
| formerForkJoinPool.shutdown(); | ||
| } | ||
| } | ||
|
|
||
| private void bounceFixedExecutor() { | ||
| if (lzFixedExecutor.isActive()) { | ||
| ExecutorService formerFixedExecutor = lzFixedExecutor.get(); | ||
| createLazyFixedExecutor(); | ||
| formerFixedExecutor.shutdown(); | ||
| } | ||
| } | ||
|
|
||
| private void bounceCtagsPool() { | ||
| if (lzCtagsPool.isActive()) { | ||
| ObjectPool<Ctags> formerCtagsPool = lzCtagsPool.get(); | ||
| createLazyCtagsPool(); | ||
| formerCtagsPool.shutdown(); | ||
| } | ||
| } | ||
|
|
||
| private void bounceHistoryExecutor() { | ||
| if (lzHistoryExecutor.isActive()) { | ||
| ExecutorService formerHistoryExecutor = lzHistoryExecutor.get(); | ||
| createLazyHistoryExecutor(); | ||
| formerHistoryExecutor.shutdown(); | ||
| } | ||
| } | ||
|
|
||
| private void bounceHistoryRenamedExecutor() { | ||
| if (lzHistoryRenamedExecutor.isActive()) { | ||
| ExecutorService formerHistoryRenamedExecutor = lzHistoryRenamedExecutor.get(); | ||
| createLazyHistoryRenamedExecutor(); | ||
| formerHistoryRenamedExecutor.shutdown(); | ||
| } | ||
| } | ||
|
|
||
| private void bounceCtagsWatcherExecutor() { | ||
| if (lzCtagsWatcherExecutor.isActive()) { | ||
| ExecutorService formerCtagsWatcherExecutor = lzCtagsWatcherExecutor.get(); | ||
| createLazyCtagsWatcherExecutor(); | ||
| formerCtagsWatcherExecutor.shutdown(); | ||
| } | ||
| } | ||
|
|
||
| /** | ||
| * Shutdown the executor used for repository search. | ||
| * @see #bounce() | ||
| */ | ||
| public void bounceRepositorySearchExecutor() { | ||
| if (lzRepositorySearchExecutor.isActive()) { | ||
| ExecutorService formerRepositorySearchExecutor = lzRepositorySearchExecutor.get(); | ||
| createLazyRepositorySearchExecutor(); | ||
| formerRepositorySearchExecutor.shutdown(); | ||
| } | ||
| } | ||
|
|
||
| private void createLazyForkJoinPool() { | ||
| lzForkJoinPool = LazilyInstantiate.using(() -> | ||
| new ForkJoinPool(indexingParallelism)); | ||
|
|
@@ -226,6 +268,11 @@ private void createLazyHistoryRenamedExecutor() { | |
| Executors.newFixedThreadPool(env.getHistoryRenamedParallelism())); | ||
| } | ||
|
|
||
| private void createLazyRepositorySearchExecutor() { | ||
| lzRepositorySearchExecutor = LazilyInstantiate.using(() -> | ||
| Executors.newFixedThreadPool(env.getRepositorySearchParallelism())); | ||
| } | ||
|
|
||
| private class CtagsObjectFactory implements ObjectFactory<Ctags> { | ||
|
|
||
| public Ctags createNew() { | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters