View Javadoc
1   /*
2    * Licensed under the Apache License, Version 2.0 (the "License");
3    * you may not use this file except in compliance with the License.
4    * You may obtain a copy of the License at
5    *
6    * http://www.apache.org/licenses/LICENSE-2.0
7    *
8    * Unless required by applicable law or agreed to in writing, software
9    * distributed under the License is distributed on an "AS IS" BASIS,
10   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11   * See the License for the specific language governing permissions and
12   * limitations under the License.
13   */
14  
15  package org.basepom.mojo.dvc.dependency;
16  
17  import static com.google.common.base.Preconditions.checkNotNull;
18  import static com.google.common.base.Preconditions.checkState;
19  import static com.google.common.collect.ImmutableList.toImmutableList;
20  import static com.google.common.collect.ImmutableSet.toImmutableSet;
21  import static java.lang.String.format;
22  import static org.basepom.mojo.dvc.dependency.DependencyMapBuilder.convertToPomArtifact;
23  
24  import org.basepom.mojo.dvc.CheckExclusionsFilter;
25  import org.basepom.mojo.dvc.Context;
26  import org.basepom.mojo.dvc.PluginLog;
27  import org.basepom.mojo.dvc.QualifiedName;
28  import org.basepom.mojo.dvc.ScopeLimitingFilter;
29  import org.basepom.mojo.dvc.strategy.Strategy;
30  import org.basepom.mojo.dvc.version.VersionResolution;
31  import org.basepom.mojo.dvc.version.VersionResolutionCollection;
32  
33  import java.time.Duration;
34  import java.util.Collection;
35  import java.util.Set;
36  import java.util.concurrent.Callable;
37  import java.util.concurrent.ExecutionException;
38  import java.util.concurrent.Executors;
39  import java.util.concurrent.locks.Lock;
40  import java.util.concurrent.locks.ReentrantLock;
41  
42  import com.google.common.base.Joiner;
43  import com.google.common.collect.ImmutableList;
44  import com.google.common.collect.ImmutableSet;
45  import com.google.common.collect.ImmutableSetMultimap;
46  import com.google.common.util.concurrent.ListenableFuture;
47  import com.google.common.util.concurrent.ListeningExecutorService;
48  import com.google.common.util.concurrent.MoreExecutors;
49  import com.google.common.util.concurrent.ThreadFactoryBuilder;
50  import org.apache.maven.RepositoryUtils;
51  import org.apache.maven.artifact.resolver.AbstractArtifactResolutionException;
52  import org.apache.maven.artifact.versioning.ComparableVersion;
53  import org.apache.maven.artifact.versioning.OverConstrainedVersionException;
54  import org.apache.maven.plugin.MojoExecutionException;
55  import org.apache.maven.project.DependencyResolutionException;
56  import org.apache.maven.project.MavenProject;
57  import org.apache.maven.project.ProjectBuildingException;
58  import org.eclipse.aether.RepositorySystem;
59  import org.eclipse.aether.artifact.Artifact;
60  import org.eclipse.aether.artifact.ArtifactTypeRegistry;
61  import org.eclipse.aether.graph.Dependency;
62  import org.eclipse.aether.graph.DependencyFilter;
63  import org.eclipse.aether.graph.DependencyNode;
64  import org.eclipse.aether.resolution.VersionRangeRequest;
65  import org.eclipse.aether.resolution.VersionRangeResolutionException;
66  import org.eclipse.aether.resolution.VersionRangeResult;
67  import org.eclipse.aether.util.filter.AndDependencyFilter;
68  
69  public final class DependencyTreeResolver
70          implements AutoCloseable {
71  
72      private static final PluginLog LOG = new PluginLog(DependencyTreeResolver.class);
73  
74      private static final int DEPENDENCY_RESOLUTION_NUM_THREADS = Runtime.getRuntime().availableProcessors() * 5;
75  
76      private final Lock collectorLock = new ReentrantLock();
77  
78      private final Context context;
79      private final DependencyMap rootDependencyMap;
80  
81      private final ListeningExecutorService executorService = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(DEPENDENCY_RESOLUTION_NUM_THREADS,
82              new ThreadFactoryBuilder().setNameFormat("dependency-version-check-worker-%s").setDaemon(true).build()));
83  
84      public DependencyTreeResolver(final Context context, final DependencyMap rootDependencyMap) {
85          this.context = checkNotNull(context, "context is null");
86          this.rootDependencyMap = checkNotNull(rootDependencyMap, "rootDependencyMap is null");
87      }
88  
89      @Override
90      public void close() {
91          MoreExecutors.shutdownAndAwaitTermination(executorService, Duration.ofSeconds(2));
92      }
93  
94      /**
95       * Creates a map of all dependency version resolutions used in this project in a given scope. The result is a map from names to a list of version numbers
96       * used in the project, based on the element requesting the version.
97       * <p>
98       * If the special scope "null" is used, a superset of all scopes is used (this is used by the check mojo).
99       *
100      * @param project     The maven project to resolve all dependencies for.
101      * @param scopeFilter Limits the scopes to resolve.
102      * @return Map from qualified names to possible version resolutions.
103      * @throws MojoExecutionException Parallel dependency resolution failed.
104      */
105     public ImmutableSetMultimap<QualifiedName, VersionResolutionCollection> computeResolutionMap(final MavenProject project,
106             final ScopeLimitingFilter scopeFilter)
107             throws MojoExecutionException {
108         checkNotNull(project, "project is null");
109         checkNotNull(scopeFilter, "scope is null");
110 
111         final ImmutableSetMultimap.Builder<QualifiedName, VersionResolution> collector = ImmutableSetMultimap.builder();
112         final ImmutableList.Builder<ListenableFuture<?>> futureBuilder = ImmutableList.builder();
113 
114         boolean useParallelDependencyResolution = context.useFastResolution();
115         // Map from dependency name --> list of resolutions found on the tree
116         LOG.debug("Using parallel dependency resolution: %s", useParallelDependencyResolution);
117 
118         final ImmutableList<Dependency> dependencies;
119         if (context.useDeepScan()) {
120             LOG.debug("Running deep scan");
121             dependencies = ImmutableList.copyOf(
122                     rootDependencyMap.getAllDependencies().values().stream().map(DependencyNode::getDependency).collect(toImmutableList()));
123         } else {
124             final ArtifactTypeRegistry stereotypes = context.getRepositorySystemSession().getArtifactTypeRegistry();
125 
126             dependencies = ImmutableList.copyOf(
127                     project.getDependencies().stream().map(d -> RepositoryUtils.toDependency(d, stereotypes)).collect(toImmutableList()));
128         }
129 
130         final ImmutableSet.Builder<Throwable> throwableBuilder = ImmutableSet.builder();
131 
132         if (useParallelDependencyResolution) {
133             for (final Dependency dependency : dependencies) {
134                 futureBuilder.add(executorService.submit((Callable<Void>) () -> {
135                     resolveProjectDependency(dependency, scopeFilter, collector);
136                     return null;
137                 }));
138             }
139 
140             final ImmutableList<ListenableFuture<?>> futures = futureBuilder.build();
141 
142             for (final ListenableFuture<?> future : futures) {
143                 try {
144                     future.get();
145                 } catch (InterruptedException e) {
146                     Thread.currentThread().interrupt();
147                 } catch (ExecutionException e) {
148                     throwableBuilder.add(e.getCause());
149                 }
150             }
151         } else {
152             for (final Dependency dependency : dependencies) {
153                 try {
154                     resolveProjectDependency(dependency, scopeFilter, collector);
155                 } catch (Exception e) {
156                     throwableBuilder.add(e);
157                 }
158             }
159         }
160 
161         final Set<Throwable> throwables = throwableBuilder.build();
162         if (!throwables.isEmpty()) {
163             throw processResolveProjectDependencyException(throwableBuilder.build());
164         }
165 
166         return VersionResolutionCollection.toResolutionMap(collector.build());
167     }
168 
169     private static MojoExecutionException processResolveProjectDependencyException(Set<Throwable> throwables) {
170         ImmutableSet.Builder<String> failedDependenciesBuilder = ImmutableSet.builder();
171         ImmutableSet.Builder<String> messageBuilder = ImmutableSet.builder();
172         for (Throwable t : throwables) {
173             if (t instanceof DependencyResolutionException) {
174                 ((DependencyResolutionException) t).getResult().getUnresolvedDependencies()
175                         .forEach(d -> failedDependenciesBuilder.add(printDependency(d)));
176             } else {
177                 messageBuilder.add(t.getMessage());
178             }
179         }
180 
181         String message = Joiner.on("    \n").join(messageBuilder.build());
182         Set<String> failedDependencies = failedDependenciesBuilder.build();
183         if (!failedDependencies.isEmpty()) {
184             if (!message.isEmpty()) {
185                 message += "\n";
186             }
187             message += "Could not resolve dependencies: [" + Joiner.on(", ").join(failedDependencies) + "]";
188         }
189         return new MojoExecutionException(message);
190     }
191 
192     private static String printDependency(Dependency d) {
193         return d.getArtifact() + " [" + d.getScope() + (d.isOptional() ? ", optional" : "") + "]";
194     }
195 
196     /**
197      * Called for any direct project dependency. Factored out from {@link #computeResolutionMap} to allow parallel evaluation of dependencies to speed up the
198      * process.
199      */
200     private void resolveProjectDependency(final Dependency dependency,
201             final ScopeLimitingFilter visibleScopes,
202             final ImmutableSetMultimap.Builder<QualifiedName, VersionResolution> collector)
203             throws MojoExecutionException, DependencyResolutionException, AbstractArtifactResolutionException, VersionRangeResolutionException {
204         final QualifiedName dependencyName = QualifiedName.fromDependency(dependency);
205 
206         // see if the resolved, direct dependency contain this name.
207         // If not, the dependency is declared in a scope that is not used (it was filtered out by the scope filter
208         // when the map was created.
209         if (rootDependencyMap.getDirectDependencies().containsKey(dependencyName)) {
210             // a direct dependency
211             final DependencyNode projectDependencyNode = rootDependencyMap.getDirectDependencies().get(dependencyName);
212             assert projectDependencyNode != null;
213 
214             checkState(visibleScopes.accept(projectDependencyNode, ImmutableList.of()),
215                     "Dependency %s maps to %s, but scope filter would exclude it. This should never happen!", dependency, projectDependencyNode);
216             computeVersionResolutionForDirectDependency(collector, dependency, projectDependencyNode);
217         }
218 
219         // could be a dependency in the full dependency list
220         final DependencyNode projectDependencyNode = rootDependencyMap.getAllDependencies().get(dependencyName);
221         // A project dependency could be e.g. in test scope, but the map has been computed in a narrower scope (e.g. compile)
222         // in that case, it does not contain a dependency node for the dependency. That is ok, simply ignore it.
223         if (projectDependencyNode == null) {
224             return;
225         }
226         checkState(visibleScopes.accept(projectDependencyNode, ImmutableList.of()),
227                 "Dependency %s maps to %s, but scope filter would exclude it. This should never happen!", dependency, projectDependencyNode);
228 
229         try {
230             // remove the test scope for resolving all the transitive dependencies. Anything that was pulled in in test scope,
231             // now needs its dependencies resolved in compile+runtime scope, not test scope.
232             final ScopeLimitingFilter dependencyScope = ScopeLimitingFilter.computeTransitiveScope(dependency.getScope());
233             computeVersionResolutionForTransitiveDependencies(collector, dependency, projectDependencyNode, dependencyScope);
234         } catch (ProjectBuildingException e) {
235             // This is an optimization and a bug workaround at the same time. Some artifacts exist that
236             // specify a packaging that is not natively supported by maven (e.g. bundle of OSGi bundles), however they
237             // do not bring the necessary extensions to deal with that type. As a result, this causes a "could not read model"
238             // exception. Ignore the transitive dependencies if the project node does not suggest any child artifacts.
239             if (projectDependencyNode.getChildren().isEmpty()) {
240                 LOG.debug("Ignoring model building exception for %s, no children were declared", dependency);
241             } else {
242                 LOG.warn("Could not read POM for %s, ignoring project and its dependencies!", dependency);
243             }
244         }
245     }
246 
247     /**
248      * Create a version resolution for the given direct requestingDependency and artifact.
249      */
250     private void computeVersionResolutionForDirectDependency(
251             final ImmutableSetMultimap.Builder<QualifiedName, VersionResolution> collector,
252             final Dependency requestingDependency,
253             final DependencyNode resolvedDependencyNode)
254             throws AbstractArtifactResolutionException, VersionRangeResolutionException, MojoExecutionException {
255         final QualifiedName requestingDependencyName = QualifiedName.fromDependency(requestingDependency);
256 
257         final RepositorySystem repoSystem = context.getRepositorySystem();
258 
259         Artifact artifact = convertToPomArtifact(requestingDependency.getArtifact());
260         if (artifact.isSnapshot()) {
261             // convert version of a snapshot artifact to be SNAPSHOT, otherwise the
262             // version range resolver will try to match the timestamp version
263             artifact = artifact.setVersion(artifact.getBaseVersion());
264         }
265 
266         final VersionRangeRequest request = context.createVersionRangeRequest(artifact);
267         final VersionRangeResult result = repoSystem.resolveVersionRange(context.getRepositorySystemSession(), request);
268 
269         if (!result.getVersions().contains(resolvedDependencyNode.getVersion())) {
270             throw new MojoExecutionException(
271                     format("Cannot determine the recommended version of dependency '%s'; its version specification is '%s', and the resolved version is '%s'.",
272                             requestingDependency, requestingDependency.getArtifact().getBaseVersion(), resolvedDependencyNode.getVersion()));
273         }
274 
275         // dependency range contains the project version (or matches it)
276 
277         // version from the dependency artifact
278         final ComparableVersion expectedVersion = getVersion(resolvedDependencyNode);
279 
280         // this is a direct dependency; it made it through the filter in resolveProjectDependency.
281         final boolean managedDependency = (resolvedDependencyNode.getManagedBits() & DependencyNode.MANAGED_VERSION) != 0;
282         final VersionResolution resolution = VersionResolution.forDirectDependency(QualifiedName.fromProject(context.getRootProject()), expectedVersion,
283                 managedDependency);
284 
285         if (isIncluded(resolvedDependencyNode, expectedVersion, expectedVersion)) {
286             final Strategy strategy = context.getStrategyCache().forQualifiedName(requestingDependencyName);
287             checkState(strategy != null, "Strategy for %s is null, this should never happen (could not find default strategy?", requestingDependencyName);
288 
289             if (!strategy.isCompatible(expectedVersion, expectedVersion)) {
290                 resolution.conflict();
291             }
292         } else {
293             LOG.debug("VersionResolution %s is excluded by configuration.", resolution);
294         }
295 
296         try {
297             collectorLock.lock();
298             collector.put(requestingDependencyName, resolution);
299         } finally {
300             collectorLock.unlock();
301         }
302     }
303 
304     /**
305      * Resolve all transitive dependencies relative to a given dependency, based off the artifact given. A scope filter can be added which limits the results to
306      * the scopes present in that filter.
307      */
308     private void computeVersionResolutionForTransitiveDependencies(
309             final ImmutableSetMultimap.Builder<QualifiedName, VersionResolution> collector,
310             final Dependency requestingDependency,
311             final DependencyNode dependencyNodeForDependency,
312             final DependencyFilter scopeFilter)
313             throws AbstractArtifactResolutionException, ProjectBuildingException, DependencyResolutionException {
314         final AndDependencyFilter filter = new AndDependencyFilter(scopeFilter, new CheckExclusionsFilter(requestingDependency.getExclusions()));
315 
316         final DependencyMap dependencyMap = new DependencyMapBuilder(context).mapDependency(dependencyNodeForDependency, filter);
317         final Collection<DependencyNode> transitiveDependencies = dependencyMap.getAllDependencies().values();
318         final QualifiedName requestingDependencyName = QualifiedName.fromDependency(requestingDependency);
319 
320         final ImmutableSet<DependencyNode> filteredDependencies = transitiveDependencies.stream()
321                 .filter(d -> scopeFilter.accept(d, ImmutableList.of()))
322                 .filter(d -> !d.getDependency().isOptional())
323                 .collect(toImmutableSet());
324 
325         for (final DependencyNode dependencyNode : filteredDependencies) {
326             final QualifiedName dependencyName = QualifiedName.fromDependencyNode(dependencyNode);
327 
328             final DependencyNode projectDependencyNode = rootDependencyMap.getAllDependencies().get(dependencyName);
329             if (projectDependencyNode == null) {
330                 // the next condition can happen if a dependency is required by one dependency but then overridden by another. e.g.
331                 //
332                 //   guava (*29.1-jre*, 25.1-android)
333                 //    guava 25.1-android depends on org.checkerframework:checker-compat-qual
334                 //    guava 29.1-jre depends on org.checkerframework:checker-qual
335                 //
336                 // as the dependency resolver chose 29.1-jre, only the "checker-qual" dependency will show on the final classpath
337                 // however, when resolving all dependencies, the code will resolve the dependency which pulls in guava-25.1-android.
338                 // For that dependency, there will be "checker-compat-qual" in the list of dependencies, but when the code tries to
339                 // resolve the actual classpath dependency, the "checker-compat-qual" dependency is not in the final classpath.
340                 //
341                 // This is normal situation and the dependency can just be dropped.
342                 //
343                 continue;
344             }
345 
346             final ComparableVersion resolvedVersion = getVersion(projectDependencyNode);
347             final ComparableVersion expectedVersion = getVersion(dependencyNode);
348 
349             final boolean managedDependency = (projectDependencyNode.getManagedBits() & DependencyNode.MANAGED_VERSION) != 0;
350             final VersionResolution resolution = VersionResolution.forTransitiveDependency(requestingDependencyName, expectedVersion, managedDependency);
351 
352             if (isIncluded(dependencyNode, expectedVersion, resolvedVersion)) {
353                 final Strategy strategy = context.getStrategyCache().forQualifiedName(dependencyName);
354                 checkState(strategy != null, "Strategy for %s is null, this should never happen (could not find default strategy?", dependencyName);
355 
356                 if (!strategy.isCompatible(expectedVersion, resolvedVersion)) {
357                     resolution.conflict();
358                 }
359             }
360 
361             try {
362                 collectorLock.lock();
363                 collector.put(dependencyName, resolution);
364             } finally {
365                 collectorLock.unlock();
366             }
367         }
368     }
369 
370     /**
371      * Returns true if a given artifact and version should be checked.
372      */
373     private boolean isIncluded(DependencyNode dependencyNodeForDependency, ComparableVersion expectedVersion, ComparableVersion resolvedVersion) {
374         return context.getExclusions().stream().noneMatch(exclusion -> exclusion.matches(dependencyNodeForDependency, expectedVersion, resolvedVersion));
375     }
376 
377     /**
378      * Return a version object for an Artifact.
379      */
380     private static ComparableVersion getVersion(DependencyNode dependencyNode)
381             throws OverConstrainedVersionException {
382         checkNotNull(dependencyNode, "dependencyNode is null");
383 
384         checkState(dependencyNode.getVersion() != null, "DependencyNode %s has a null version selected. Please report a bug!", dependencyNode);
385         return new ComparableVersion(dependencyNode.getVersion().toString());
386     }
387 }