View Javadoc
1   /*
2    * Licensed under the Apache License, Version 2.0 (the "License");
3    * you may not use this file except in compliance with the License.
4    * You may obtain a copy of the License at
5    *
6    * http://www.apache.org/licenses/LICENSE-2.0
7    *
8    * Unless required by applicable law or agreed to in writing, software
9    * distributed under the License is distributed on an "AS IS" BASIS,
10   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11   * See the License for the specific language governing permissions and
12   * limitations under the License.
13   */
14  
15  package org.basepom.mojo.dvc.dependency;
16  
17  import static com.google.common.base.Preconditions.checkNotNull;
18  import static com.google.common.base.Preconditions.checkState;
19  import static com.google.common.collect.ImmutableList.toImmutableList;
20  import static com.google.common.collect.ImmutableSet.toImmutableSet;
21  import static java.lang.String.format;
22  import static org.basepom.mojo.dvc.dependency.DependencyMapBuilder.convertToPomArtifact;
23  
24  import org.basepom.mojo.dvc.CheckExclusionsFilter;
25  import org.basepom.mojo.dvc.Context;
26  import org.basepom.mojo.dvc.PluginLog;
27  import org.basepom.mojo.dvc.QualifiedName;
28  import org.basepom.mojo.dvc.ScopeLimitingFilter;
29  import org.basepom.mojo.dvc.strategy.Strategy;
30  import org.basepom.mojo.dvc.version.VersionResolution;
31  import org.basepom.mojo.dvc.version.VersionResolutionCollection;
32  
33  import java.time.Duration;
34  import java.util.Collection;
35  import java.util.Set;
36  import java.util.concurrent.Callable;
37  import java.util.concurrent.ExecutionException;
38  import java.util.concurrent.Executors;
39  
40  import com.google.common.base.Joiner;
41  import com.google.common.collect.ImmutableList;
42  import com.google.common.collect.ImmutableSet;
43  import com.google.common.collect.ImmutableSetMultimap;
44  import com.google.common.util.concurrent.ListenableFuture;
45  import com.google.common.util.concurrent.ListeningExecutorService;
46  import com.google.common.util.concurrent.MoreExecutors;
47  import com.google.common.util.concurrent.ThreadFactoryBuilder;
48  import org.apache.maven.RepositoryUtils;
49  import org.apache.maven.artifact.resolver.AbstractArtifactResolutionException;
50  import org.apache.maven.artifact.versioning.ComparableVersion;
51  import org.apache.maven.artifact.versioning.OverConstrainedVersionException;
52  import org.apache.maven.plugin.MojoExecutionException;
53  import org.apache.maven.project.DependencyResolutionException;
54  import org.apache.maven.project.MavenProject;
55  import org.apache.maven.project.ProjectBuildingException;
56  import org.eclipse.aether.RepositorySystem;
57  import org.eclipse.aether.artifact.Artifact;
58  import org.eclipse.aether.artifact.ArtifactTypeRegistry;
59  import org.eclipse.aether.graph.Dependency;
60  import org.eclipse.aether.graph.DependencyFilter;
61  import org.eclipse.aether.graph.DependencyNode;
62  import org.eclipse.aether.resolution.VersionRangeRequest;
63  import org.eclipse.aether.resolution.VersionRangeResolutionException;
64  import org.eclipse.aether.resolution.VersionRangeResult;
65  import org.eclipse.aether.util.filter.AndDependencyFilter;
66  
67  public final class DependencyTreeResolver
68          implements AutoCloseable {
69  
70      private static final PluginLog LOG = new PluginLog(DependencyTreeResolver.class);
71  
72      private static final int DEPENDENCY_RESOLUTION_NUM_THREADS = Runtime.getRuntime().availableProcessors() * 5;
73  
74      private final Context context;
75      private final DependencyMap rootDependencyMap;
76  
77      private final ListeningExecutorService executorService = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(DEPENDENCY_RESOLUTION_NUM_THREADS,
78              new ThreadFactoryBuilder().setNameFormat("dependency-version-check-worker-%s").setDaemon(true).build()));
79  
80      public DependencyTreeResolver(final Context context, final DependencyMap rootDependencyMap) {
81          this.context = checkNotNull(context, "context is null");
82          this.rootDependencyMap = checkNotNull(rootDependencyMap, "rootDependencyMap is null");
83      }
84  
85      @Override
86      public void close() {
87          MoreExecutors.shutdownAndAwaitTermination(executorService, Duration.ofSeconds(2));
88      }
89  
90      /**
91       * Creates a map of all dependency version resolutions used in this project in a given scope. The result is a map from names to a list of version numbers
92       * used in the project, based on the element requesting the version.
93       * <p>
94       * If the special scope "null" is used, a superset of all scopes is used (this is used by the check mojo).
95       *
96       * @param project     The maven project to resolve all dependencies for.
97       * @param scopeFilter Limits the scopes to resolve.
98       * @return Map from qualified names to possible version resolutions.
99       * @throws MojoExecutionException Parallel dependency resolution failed.
100      */
101     public ImmutableSetMultimap<QualifiedName, VersionResolutionCollection> computeResolutionMap(final MavenProject project,
102             final ScopeLimitingFilter scopeFilter)
103             throws MojoExecutionException {
104         checkNotNull(project, "project is null");
105         checkNotNull(scopeFilter, "scope is null");
106 
107         final ImmutableSetMultimap.Builder<QualifiedName, VersionResolution> collector = ImmutableSetMultimap.builder();
108         final ImmutableList.Builder<ListenableFuture<?>> futureBuilder = ImmutableList.builder();
109 
110         boolean useParallelDependencyResolution = context.useFastResolution();
111         // Map from dependency name --> list of resolutions found on the tree
112         LOG.debug("Using parallel dependency resolution: %s", useParallelDependencyResolution);
113 
114         final ImmutableList<Dependency> dependencies;
115         if (context.useDeepScan()) {
116             LOG.debug("Running deep scan");
117             dependencies = ImmutableList.copyOf(
118                     rootDependencyMap.getAllDependencies().values().stream().map(DependencyNode::getDependency).collect(toImmutableList()));
119         } else {
120             final ArtifactTypeRegistry stereotypes = context.getRepositorySystemSession().getArtifactTypeRegistry();
121 
122             dependencies = ImmutableList.copyOf(
123                     project.getDependencies().stream().map(d -> RepositoryUtils.toDependency(d, stereotypes)).collect(toImmutableList()));
124         }
125 
126         final ImmutableSet.Builder<Throwable> throwableBuilder = ImmutableSet.builder();
127 
128         if (useParallelDependencyResolution) {
129             for (final Dependency dependency : dependencies) {
130                 futureBuilder.add(executorService.submit((Callable<Void>) () -> {
131                     resolveProjectDependency(dependency, scopeFilter, collector);
132                     return null;
133                 }));
134             }
135 
136             final ImmutableList<ListenableFuture<?>> futures = futureBuilder.build();
137 
138             for (final ListenableFuture<?> future : futures) {
139                 try {
140                     future.get();
141                 } catch (InterruptedException e) {
142                     Thread.currentThread().interrupt();
143                 } catch (ExecutionException e) {
144                     throwableBuilder.add(e.getCause());
145                 }
146             }
147         } else {
148             for (final Dependency dependency : dependencies) {
149                 try {
150                     resolveProjectDependency(dependency, scopeFilter, collector);
151                 } catch (Throwable t) {
152                     throwableBuilder.add(t);
153                 }
154             }
155         }
156 
157         final Set<Throwable> throwables = throwableBuilder.build();
158         if (!throwables.isEmpty()) {
159             throw processResolveProjectDependencyException(throwableBuilder.build());
160         }
161 
162         return VersionResolutionCollection.toResolutionMap(collector.build());
163     }
164 
165     private static MojoExecutionException processResolveProjectDependencyException(Set<Throwable> throwables) {
166         ImmutableSet.Builder<String> failedDependenciesBuilder = ImmutableSet.builder();
167         ImmutableSet.Builder<String> messageBuilder = ImmutableSet.builder();
168         for (Throwable t : throwables) {
169             if (t instanceof DependencyResolutionException) {
170                 ((DependencyResolutionException) t).getResult().getUnresolvedDependencies()
171                         .forEach(d -> failedDependenciesBuilder.add(printDependency(d)));
172             } else {
173                 messageBuilder.add(t.getMessage());
174             }
175         }
176 
177         String message = Joiner.on("    \n").join(messageBuilder.build());
178         Set<String> failedDependencies = failedDependenciesBuilder.build();
179         if (!failedDependencies.isEmpty()) {
180             if (!message.isEmpty()) {
181                 message += "\n";
182             }
183             message += "Could not resolve dependencies: [" + Joiner.on(", ").join(failedDependencies) + "]";
184         }
185         return new MojoExecutionException(message);
186     }
187 
188     private static String printDependency(Dependency d) {
189         return d.getArtifact() + " [" + d.getScope() + (d.isOptional() ? ", optional" : "") + "]";
190     }
191 
192     /**
193      * Called for any direct project dependency. Factored out from {@link #computeResolutionMap} to allow parallel evaluation of dependencies to speed up the
194      * process.
195      */
196     private void resolveProjectDependency(final Dependency dependency,
197             final ScopeLimitingFilter visibleScopes,
198             final ImmutableSetMultimap.Builder<QualifiedName, VersionResolution> collector)
199             throws MojoExecutionException, DependencyResolutionException, AbstractArtifactResolutionException, VersionRangeResolutionException {
200         final QualifiedName dependencyName = QualifiedName.fromDependency(dependency);
201 
202         // see if the resolved, direct dependency contain this name.
203         // If not, the dependency is declared in a scope that is not used (it was filtered out by the scope filter
204         // when the map was created.
205         if (rootDependencyMap.getDirectDependencies().containsKey(dependencyName)) {
206             // a direct dependency
207             final DependencyNode projectDependencyNode = rootDependencyMap.getDirectDependencies().get(dependencyName);
208             assert projectDependencyNode != null;
209 
210             checkState(visibleScopes.accept(projectDependencyNode, ImmutableList.of()),
211                     "Dependency %s maps to %s, but scope filter would exclude it. This should never happen!", dependency, projectDependencyNode);
212             computeVersionResolutionForDirectDependency(collector, dependency, projectDependencyNode);
213         }
214 
215         // could be a dependency in the full dependency list
216         final DependencyNode projectDependencyNode = rootDependencyMap.getAllDependencies().get(dependencyName);
217         // A project dependency could be e.g. in test scope, but the map has been computed in a narrower scope (e.g. compile)
218         // in that case, it does not contain a dependency node for the dependency. That is ok, simply ignore it.
219         if (projectDependencyNode == null) {
220             return;
221         }
222         checkState(visibleScopes.accept(projectDependencyNode, ImmutableList.of()),
223                 "Dependency %s maps to %s, but scope filter would exclude it. This should never happen!", dependency, projectDependencyNode);
224 
225         try {
226             // remove the test scope for resolving all the transitive dependencies. Anything that was pulled in in test scope,
227             // now needs its dependencies resolved in compile+runtime scope, not test scope.
228             final ScopeLimitingFilter dependencyScope = ScopeLimitingFilter.computeTransitiveScope(dependency.getScope());
229             computeVersionResolutionForTransitiveDependencies(collector, dependency, projectDependencyNode, dependencyScope);
230         } catch (ProjectBuildingException e) {
231             // This is an optimization and a bug workaround at the same time. Some artifacts exist that
232             // specify a packaging that is not natively supported by maven (e.g. bundle of OSGi bundles), however they
233             // do not bring the necessary extensions to deal with that type. As a result, this causes a "could not read model"
234             // exception. Ignore the transitive dependencies if the project node does not suggest any child artifacts.
235             if (projectDependencyNode.getChildren().isEmpty()) {
236                 LOG.debug("Ignoring model building exception for %s, no children were declared", dependency);
237             } else {
238                 LOG.warn("Could not read POM for %s, ignoring project and its dependencies!", dependency);
239             }
240         }
241     }
242 
243     /**
244      * Create a version resolution for the given direct requestingDependency and artifact.
245      */
246     private void computeVersionResolutionForDirectDependency(
247             final ImmutableSetMultimap.Builder<QualifiedName, VersionResolution> collector,
248             final Dependency requestingDependency,
249             final DependencyNode resolvedDependencyNode)
250             throws AbstractArtifactResolutionException, VersionRangeResolutionException, MojoExecutionException {
251         final QualifiedName requestingDependencyName = QualifiedName.fromDependency(requestingDependency);
252 
253         final RepositorySystem repoSystem = context.getRepositorySystem();
254 
255         Artifact artifact = convertToPomArtifact(requestingDependency.getArtifact());
256         if (artifact.isSnapshot()) {
257             // convert version of a snapshot artifact to be SNAPSHOT, otherwise the
258             // version range resolver will try to match the timestamp version
259             artifact = artifact.setVersion(artifact.getBaseVersion());
260         }
261 
262         final VersionRangeRequest request = context.createVersionRangeRequest(artifact);
263         final VersionRangeResult result = repoSystem.resolveVersionRange(context.getRepositorySystemSession(), request);
264 
265         if (!result.getVersions().contains(resolvedDependencyNode.getVersion())) {
266             throw new MojoExecutionException(
267                     format("Cannot determine the recommended version of dependency '%s'; its version specification is '%s', and the resolved version is '%s'.",
268                             requestingDependency, requestingDependency.getArtifact().getBaseVersion(), resolvedDependencyNode.getVersion()));
269         }
270 
271         // dependency range contains the project version (or matches it)
272 
273         // version from the dependency artifact
274         final ComparableVersion expectedVersion = getVersion(resolvedDependencyNode);
275 
276         // this is a direct dependency; it made it through the filter in resolveProjectDependency.
277         final boolean managedDependency = (resolvedDependencyNode.getManagedBits() & DependencyNode.MANAGED_VERSION) != 0;
278         final VersionResolution resolution = VersionResolution.forDirectDependency(QualifiedName.fromProject(context.getRootProject()), expectedVersion,
279                 managedDependency);
280 
281         if (isIncluded(resolvedDependencyNode, expectedVersion, expectedVersion)) {
282             final Strategy strategy = context.getStrategyCache().forQualifiedName(requestingDependencyName);
283             checkState(strategy != null, "Strategy for %s is null, this should never happen (could not find default strategy?", requestingDependencyName);
284 
285             if (!strategy.isCompatible(expectedVersion, expectedVersion)) {
286                 resolution.conflict();
287             }
288         } else {
289             LOG.debug("VersionResolution %s is excluded by configuration.", resolution);
290         }
291 
292         synchronized (collector) {
293             collector.put(requestingDependencyName, resolution);
294         }
295     }
296 
297     /**
298      * Resolve all transitive dependencies relative to a given dependency, based off the artifact given. A scope filter can be added which limits the results to
299      * the scopes present in that filter.
300      */
301     private void computeVersionResolutionForTransitiveDependencies(
302             final ImmutableSetMultimap.Builder<QualifiedName, VersionResolution> collector,
303             final Dependency requestingDependency,
304             final DependencyNode dependencyNodeForDependency,
305             final DependencyFilter scopeFilter)
306             throws AbstractArtifactResolutionException, ProjectBuildingException, DependencyResolutionException {
307         final AndDependencyFilter filter = new AndDependencyFilter(scopeFilter, new CheckExclusionsFilter(requestingDependency.getExclusions()));
308 
309         final DependencyMap dependencyMap = new DependencyMapBuilder(context).mapDependency(dependencyNodeForDependency, filter);
310         final Collection<DependencyNode> transitiveDependencies = dependencyMap.getAllDependencies().values();
311         final QualifiedName requestingDependencyName = QualifiedName.fromDependency(requestingDependency);
312 
313         final ImmutableSet<DependencyNode> filteredDependencies = transitiveDependencies.stream()
314                 .filter(d -> scopeFilter.accept(d, ImmutableList.of()))
315                 .filter(d -> !d.getDependency().isOptional())
316                 .collect(toImmutableSet());
317 
318         for (final DependencyNode dependencyNode : filteredDependencies) {
319             final QualifiedName dependencyName = QualifiedName.fromDependencyNode(dependencyNode);
320 
321             final DependencyNode projectDependencyNode = rootDependencyMap.getAllDependencies().get(dependencyName);
322             if (projectDependencyNode == null) {
323                 // the next condition can happen if a dependency is required by one dependency but then overridden by another. e.g.
324                 //
325                 //   guava (*29.1-jre*, 25.1-android)
326                 //    guava 25.1-android depends on org.checkerframework:checker-compat-qual
327                 //    guava 29.1-jre depends on org.checkerframework:checker-qual
328                 //
329                 // as the dependency resolver chose 29.1-jre, only the "checker-qual" dependency will show on the final classpath
330                 // however, when resolving all dependencies, the code will resolve the dependency which pulls in guava-25.1-android.
331                 // For that dependency, there will be "checker-compat-qual" in the list of dependencies, but when the code tries to
332                 // resolve the actual classpath dependency, the "checker-compat-qual" dependency is not in the final classpath.
333                 //
334                 // This is normal situation and the dependency can just be dropped.
335                 //
336                 continue;
337             }
338 
339             final ComparableVersion resolvedVersion = getVersion(projectDependencyNode);
340             final ComparableVersion expectedVersion = getVersion(dependencyNode);
341 
342             final boolean managedDependency = (projectDependencyNode.getManagedBits() & DependencyNode.MANAGED_VERSION) != 0;
343             final VersionResolution resolution = VersionResolution.forTransitiveDependency(requestingDependencyName, expectedVersion, managedDependency);
344 
345             if (isIncluded(dependencyNode, expectedVersion, resolvedVersion)) {
346                 final Strategy strategy = context.getStrategyCache().forQualifiedName(dependencyName);
347                 checkState(strategy != null, "Strategy for %s is null, this should never happen (could not find default strategy?", dependencyName);
348 
349                 if (!strategy.isCompatible(expectedVersion, resolvedVersion)) {
350                     resolution.conflict();
351                 }
352             }
353             synchronized (collector) {
354                 collector.put(dependencyName, resolution);
355             }
356         }
357     }
358 
359     /**
360      * Returns true if a given artifact and version should be checked.
361      */
362     private boolean isIncluded(DependencyNode dependencyNodeForDependency, ComparableVersion expectedVersion, ComparableVersion resolvedVersion) {
363         return context.getExclusions().stream().noneMatch(exclusion -> exclusion.matches(dependencyNodeForDependency, expectedVersion, resolvedVersion));
364     }
365 
366     /**
367      * Return a version object for an Artifact.
368      */
369     private static ComparableVersion getVersion(DependencyNode dependencyNode)
370             throws OverConstrainedVersionException {
371         checkNotNull(dependencyNode, "dependencyNode is null");
372 
373         checkState(dependencyNode.getVersion() != null, "DependencyNode %s has a null version selected. Please report a bug!", dependencyNode);
374         return new ComparableVersion(dependencyNode.getVersion().toString());
375     }
376 }