001/* 002 * Licensed under the Apache License, Version 2.0 (the "License"); 003 * you may not use this file except in compliance with the License. 004 * You may obtain a copy of the License at 005 * 006 * http://www.apache.org/licenses/LICENSE-2.0 007 * 008 * Unless required by applicable law or agreed to in writing, software 009 * distributed under the License is distributed on an "AS IS" BASIS, 010 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 011 * See the License for the specific language governing permissions and 012 * limitations under the License. 013 */ 014 015package org.basepom.mojo.dvc.dependency; 016 017import static com.google.common.base.Preconditions.checkNotNull; 018import static com.google.common.base.Preconditions.checkState; 019import static com.google.common.collect.ImmutableList.toImmutableList; 020import static com.google.common.collect.ImmutableSet.toImmutableSet; 021import static java.lang.String.format; 022import static org.basepom.mojo.dvc.dependency.DependencyMapBuilder.convertToPomArtifact; 023 024import org.basepom.mojo.dvc.CheckExclusionsFilter; 025import org.basepom.mojo.dvc.Context; 026import org.basepom.mojo.dvc.PluginLog; 027import org.basepom.mojo.dvc.QualifiedName; 028import org.basepom.mojo.dvc.ScopeLimitingFilter; 029import org.basepom.mojo.dvc.strategy.Strategy; 030import org.basepom.mojo.dvc.version.VersionResolution; 031import org.basepom.mojo.dvc.version.VersionResolutionCollection; 032 033import java.time.Duration; 034import java.util.Collection; 035import java.util.Set; 036import java.util.concurrent.Callable; 037import java.util.concurrent.ExecutionException; 038import java.util.concurrent.Executors; 039import java.util.concurrent.locks.Lock; 040import java.util.concurrent.locks.ReentrantLock; 041 042import com.google.common.base.Joiner; 043import com.google.common.collect.ImmutableList; 044import com.google.common.collect.ImmutableSet; 045import com.google.common.collect.ImmutableSetMultimap; 046import com.google.common.util.concurrent.ListenableFuture; 047import com.google.common.util.concurrent.ListeningExecutorService; 048import com.google.common.util.concurrent.MoreExecutors; 049import com.google.common.util.concurrent.ThreadFactoryBuilder; 050import org.apache.maven.RepositoryUtils; 051import org.apache.maven.artifact.resolver.AbstractArtifactResolutionException; 052import org.apache.maven.artifact.versioning.ComparableVersion; 053import org.apache.maven.artifact.versioning.OverConstrainedVersionException; 054import org.apache.maven.plugin.MojoExecutionException; 055import org.apache.maven.project.DependencyResolutionException; 056import org.apache.maven.project.MavenProject; 057import org.apache.maven.project.ProjectBuildingException; 058import org.eclipse.aether.RepositorySystem; 059import org.eclipse.aether.artifact.Artifact; 060import org.eclipse.aether.artifact.ArtifactTypeRegistry; 061import org.eclipse.aether.graph.Dependency; 062import org.eclipse.aether.graph.DependencyFilter; 063import org.eclipse.aether.graph.DependencyNode; 064import org.eclipse.aether.resolution.VersionRangeRequest; 065import org.eclipse.aether.resolution.VersionRangeResolutionException; 066import org.eclipse.aether.resolution.VersionRangeResult; 067import org.eclipse.aether.util.filter.AndDependencyFilter; 068 069public final class DependencyTreeResolver 070 implements AutoCloseable { 071 072 private static final PluginLog LOG = new PluginLog(DependencyTreeResolver.class); 073 074 private static final int DEPENDENCY_RESOLUTION_NUM_THREADS = Runtime.getRuntime().availableProcessors() * 5; 075 076 private final Lock collectorLock = new ReentrantLock(); 077 078 private final Context context; 079 private final DependencyMap rootDependencyMap; 080 081 private final ListeningExecutorService executorService = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(DEPENDENCY_RESOLUTION_NUM_THREADS, 082 new ThreadFactoryBuilder().setNameFormat("dependency-version-check-worker-%s").setDaemon(true).build())); 083 084 public DependencyTreeResolver(final Context context, final DependencyMap rootDependencyMap) { 085 this.context = checkNotNull(context, "context is null"); 086 this.rootDependencyMap = checkNotNull(rootDependencyMap, "rootDependencyMap is null"); 087 } 088 089 @Override 090 public void close() { 091 MoreExecutors.shutdownAndAwaitTermination(executorService, Duration.ofSeconds(2)); 092 } 093 094 /** 095 * Creates a map of all dependency version resolutions used in this project in a given scope. The result is a map from names to a list of version numbers 096 * used in the project, based on the element requesting the version. 097 * <p> 098 * If the special scope "null" is used, a superset of all scopes is used (this is used by the check mojo). 099 * 100 * @param project The maven project to resolve all dependencies for. 101 * @param scopeFilter Limits the scopes to resolve. 102 * @return Map from qualified names to possible version resolutions. 103 * @throws MojoExecutionException Parallel dependency resolution failed. 104 */ 105 public ImmutableSetMultimap<QualifiedName, VersionResolutionCollection> computeResolutionMap(final MavenProject project, 106 final ScopeLimitingFilter scopeFilter) 107 throws MojoExecutionException { 108 checkNotNull(project, "project is null"); 109 checkNotNull(scopeFilter, "scope is null"); 110 111 final ImmutableSetMultimap.Builder<QualifiedName, VersionResolution> collector = ImmutableSetMultimap.builder(); 112 final ImmutableList.Builder<ListenableFuture<?>> futureBuilder = ImmutableList.builder(); 113 114 boolean useParallelDependencyResolution = context.useFastResolution(); 115 // Map from dependency name --> list of resolutions found on the tree 116 LOG.debug("Using parallel dependency resolution: %s", useParallelDependencyResolution); 117 118 final ImmutableList<Dependency> dependencies; 119 if (context.useDeepScan()) { 120 LOG.debug("Running deep scan"); 121 dependencies = ImmutableList.copyOf( 122 rootDependencyMap.getAllDependencies().values().stream().map(DependencyNode::getDependency).collect(toImmutableList())); 123 } else { 124 final ArtifactTypeRegistry stereotypes = context.getRepositorySystemSession().getArtifactTypeRegistry(); 125 126 dependencies = ImmutableList.copyOf( 127 project.getDependencies().stream().map(d -> RepositoryUtils.toDependency(d, stereotypes)).collect(toImmutableList())); 128 } 129 130 final ImmutableSet.Builder<Throwable> throwableBuilder = ImmutableSet.builder(); 131 132 if (useParallelDependencyResolution) { 133 for (final Dependency dependency : dependencies) { 134 futureBuilder.add(executorService.submit((Callable<Void>) () -> { 135 resolveProjectDependency(dependency, scopeFilter, collector); 136 return null; 137 })); 138 } 139 140 final ImmutableList<ListenableFuture<?>> futures = futureBuilder.build(); 141 142 for (final ListenableFuture<?> future : futures) { 143 try { 144 future.get(); 145 } catch (InterruptedException e) { 146 Thread.currentThread().interrupt(); 147 } catch (ExecutionException e) { 148 throwableBuilder.add(e.getCause()); 149 } 150 } 151 } else { 152 for (final Dependency dependency : dependencies) { 153 try { 154 resolveProjectDependency(dependency, scopeFilter, collector); 155 } catch (Exception e) { 156 throwableBuilder.add(e); 157 } 158 } 159 } 160 161 final Set<Throwable> throwables = throwableBuilder.build(); 162 if (!throwables.isEmpty()) { 163 throw processResolveProjectDependencyException(throwableBuilder.build()); 164 } 165 166 return VersionResolutionCollection.toResolutionMap(collector.build()); 167 } 168 169 private static MojoExecutionException processResolveProjectDependencyException(Set<Throwable> throwables) { 170 ImmutableSet.Builder<String> failedDependenciesBuilder = ImmutableSet.builder(); 171 ImmutableSet.Builder<String> messageBuilder = ImmutableSet.builder(); 172 for (Throwable t : throwables) { 173 if (t instanceof DependencyResolutionException) { 174 ((DependencyResolutionException) t).getResult().getUnresolvedDependencies() 175 .forEach(d -> failedDependenciesBuilder.add(printDependency(d))); 176 } else { 177 messageBuilder.add(t.getMessage()); 178 } 179 } 180 181 String message = Joiner.on(" \n").join(messageBuilder.build()); 182 Set<String> failedDependencies = failedDependenciesBuilder.build(); 183 if (!failedDependencies.isEmpty()) { 184 if (!message.isEmpty()) { 185 message += "\n"; 186 } 187 message += "Could not resolve dependencies: [" + Joiner.on(", ").join(failedDependencies) + "]"; 188 } 189 return new MojoExecutionException(message); 190 } 191 192 private static String printDependency(Dependency d) { 193 return d.getArtifact() + " [" + d.getScope() + (d.isOptional() ? ", optional" : "") + "]"; 194 } 195 196 /** 197 * Called for any direct project dependency. Factored out from {@link #computeResolutionMap} to allow parallel evaluation of dependencies to speed up the 198 * process. 199 */ 200 private void resolveProjectDependency(final Dependency dependency, 201 final ScopeLimitingFilter visibleScopes, 202 final ImmutableSetMultimap.Builder<QualifiedName, VersionResolution> collector) 203 throws MojoExecutionException, DependencyResolutionException, AbstractArtifactResolutionException, VersionRangeResolutionException { 204 final QualifiedName dependencyName = QualifiedName.fromDependency(dependency); 205 206 // see if the resolved, direct dependency contain this name. 207 // If not, the dependency is declared in a scope that is not used (it was filtered out by the scope filter 208 // when the map was created. 209 if (rootDependencyMap.getDirectDependencies().containsKey(dependencyName)) { 210 // a direct dependency 211 final DependencyNode projectDependencyNode = rootDependencyMap.getDirectDependencies().get(dependencyName); 212 assert projectDependencyNode != null; 213 214 checkState(visibleScopes.accept(projectDependencyNode, ImmutableList.of()), 215 "Dependency %s maps to %s, but scope filter would exclude it. This should never happen!", dependency, projectDependencyNode); 216 computeVersionResolutionForDirectDependency(collector, dependency, projectDependencyNode); 217 } 218 219 // could be a dependency in the full dependency list 220 final DependencyNode projectDependencyNode = rootDependencyMap.getAllDependencies().get(dependencyName); 221 // A project dependency could be e.g. in test scope, but the map has been computed in a narrower scope (e.g. compile) 222 // in that case, it does not contain a dependency node for the dependency. That is ok, simply ignore it. 223 if (projectDependencyNode == null) { 224 return; 225 } 226 checkState(visibleScopes.accept(projectDependencyNode, ImmutableList.of()), 227 "Dependency %s maps to %s, but scope filter would exclude it. This should never happen!", dependency, projectDependencyNode); 228 229 try { 230 // remove the test scope for resolving all the transitive dependencies. Anything that was pulled in in test scope, 231 // now needs its dependencies resolved in compile+runtime scope, not test scope. 232 final ScopeLimitingFilter dependencyScope = ScopeLimitingFilter.computeTransitiveScope(dependency.getScope()); 233 computeVersionResolutionForTransitiveDependencies(collector, dependency, projectDependencyNode, dependencyScope); 234 } catch (ProjectBuildingException e) { 235 // This is an optimization and a bug workaround at the same time. Some artifacts exist that 236 // specify a packaging that is not natively supported by maven (e.g. bundle of OSGi bundles), however they 237 // do not bring the necessary extensions to deal with that type. As a result, this causes a "could not read model" 238 // exception. Ignore the transitive dependencies if the project node does not suggest any child artifacts. 239 if (projectDependencyNode.getChildren().isEmpty()) { 240 LOG.debug("Ignoring model building exception for %s, no children were declared", dependency); 241 } else { 242 LOG.warn("Could not read POM for %s, ignoring project and its dependencies!", dependency); 243 } 244 } 245 } 246 247 /** 248 * Create a version resolution for the given direct requestingDependency and artifact. 249 */ 250 private void computeVersionResolutionForDirectDependency( 251 final ImmutableSetMultimap.Builder<QualifiedName, VersionResolution> collector, 252 final Dependency requestingDependency, 253 final DependencyNode resolvedDependencyNode) 254 throws AbstractArtifactResolutionException, VersionRangeResolutionException, MojoExecutionException { 255 final QualifiedName requestingDependencyName = QualifiedName.fromDependency(requestingDependency); 256 257 final RepositorySystem repoSystem = context.getRepositorySystem(); 258 259 Artifact artifact = convertToPomArtifact(requestingDependency.getArtifact()); 260 if (artifact.isSnapshot()) { 261 // convert version of a snapshot artifact to be SNAPSHOT, otherwise the 262 // version range resolver will try to match the timestamp version 263 artifact = artifact.setVersion(artifact.getBaseVersion()); 264 } 265 266 final VersionRangeRequest request = context.createVersionRangeRequest(artifact); 267 final VersionRangeResult result = repoSystem.resolveVersionRange(context.getRepositorySystemSession(), request); 268 269 if (!result.getVersions().contains(resolvedDependencyNode.getVersion())) { 270 throw new MojoExecutionException( 271 format("Cannot determine the recommended version of dependency '%s'; its version specification is '%s', and the resolved version is '%s'.", 272 requestingDependency, requestingDependency.getArtifact().getBaseVersion(), resolvedDependencyNode.getVersion())); 273 } 274 275 // dependency range contains the project version (or matches it) 276 277 // version from the dependency artifact 278 final ComparableVersion expectedVersion = getVersion(resolvedDependencyNode); 279 280 // this is a direct dependency; it made it through the filter in resolveProjectDependency. 281 final boolean managedDependency = (resolvedDependencyNode.getManagedBits() & DependencyNode.MANAGED_VERSION) != 0; 282 final VersionResolution resolution = VersionResolution.forDirectDependency(QualifiedName.fromProject(context.getRootProject()), expectedVersion, 283 managedDependency); 284 285 if (isIncluded(resolvedDependencyNode, expectedVersion, expectedVersion)) { 286 final Strategy strategy = context.getStrategyCache().forQualifiedName(requestingDependencyName); 287 checkState(strategy != null, "Strategy for %s is null, this should never happen (could not find default strategy?", requestingDependencyName); 288 289 if (!strategy.isCompatible(expectedVersion, expectedVersion)) { 290 resolution.conflict(); 291 } 292 } else { 293 LOG.debug("VersionResolution %s is excluded by configuration.", resolution); 294 } 295 296 try { 297 collectorLock.lock(); 298 collector.put(requestingDependencyName, resolution); 299 } finally { 300 collectorLock.unlock(); 301 } 302 } 303 304 /** 305 * Resolve all transitive dependencies relative to a given dependency, based off the artifact given. A scope filter can be added which limits the results to 306 * the scopes present in that filter. 307 */ 308 private void computeVersionResolutionForTransitiveDependencies( 309 final ImmutableSetMultimap.Builder<QualifiedName, VersionResolution> collector, 310 final Dependency requestingDependency, 311 final DependencyNode dependencyNodeForDependency, 312 final DependencyFilter scopeFilter) 313 throws AbstractArtifactResolutionException, ProjectBuildingException, DependencyResolutionException { 314 final AndDependencyFilter filter = new AndDependencyFilter(scopeFilter, new CheckExclusionsFilter(requestingDependency.getExclusions())); 315 316 final DependencyMap dependencyMap = new DependencyMapBuilder(context).mapDependency(dependencyNodeForDependency, filter); 317 final Collection<DependencyNode> transitiveDependencies = dependencyMap.getAllDependencies().values(); 318 final QualifiedName requestingDependencyName = QualifiedName.fromDependency(requestingDependency); 319 320 final ImmutableSet<DependencyNode> filteredDependencies = transitiveDependencies.stream() 321 .filter(d -> scopeFilter.accept(d, ImmutableList.of())) 322 .filter(d -> !d.getDependency().isOptional()) 323 .collect(toImmutableSet()); 324 325 for (final DependencyNode dependencyNode : filteredDependencies) { 326 final QualifiedName dependencyName = QualifiedName.fromDependencyNode(dependencyNode); 327 328 final DependencyNode projectDependencyNode = rootDependencyMap.getAllDependencies().get(dependencyName); 329 if (projectDependencyNode == null) { 330 // the next condition can happen if a dependency is required by one dependency but then overridden by another. e.g. 331 // 332 // guava (*29.1-jre*, 25.1-android) 333 // guava 25.1-android depends on org.checkerframework:checker-compat-qual 334 // guava 29.1-jre depends on org.checkerframework:checker-qual 335 // 336 // as the dependency resolver chose 29.1-jre, only the "checker-qual" dependency will show on the final classpath 337 // however, when resolving all dependencies, the code will resolve the dependency which pulls in guava-25.1-android. 338 // For that dependency, there will be "checker-compat-qual" in the list of dependencies, but when the code tries to 339 // resolve the actual classpath dependency, the "checker-compat-qual" dependency is not in the final classpath. 340 // 341 // This is normal situation and the dependency can just be dropped. 342 // 343 continue; 344 } 345 346 final ComparableVersion resolvedVersion = getVersion(projectDependencyNode); 347 final ComparableVersion expectedVersion = getVersion(dependencyNode); 348 349 final boolean managedDependency = (projectDependencyNode.getManagedBits() & DependencyNode.MANAGED_VERSION) != 0; 350 final VersionResolution resolution = VersionResolution.forTransitiveDependency(requestingDependencyName, expectedVersion, managedDependency); 351 352 if (isIncluded(dependencyNode, expectedVersion, resolvedVersion)) { 353 final Strategy strategy = context.getStrategyCache().forQualifiedName(dependencyName); 354 checkState(strategy != null, "Strategy for %s is null, this should never happen (could not find default strategy?", dependencyName); 355 356 if (!strategy.isCompatible(expectedVersion, resolvedVersion)) { 357 resolution.conflict(); 358 } 359 } 360 361 try { 362 collectorLock.lock(); 363 collector.put(dependencyName, resolution); 364 } finally { 365 collectorLock.unlock(); 366 } 367 } 368 } 369 370 /** 371 * Returns true if a given artifact and version should be checked. 372 */ 373 private boolean isIncluded(DependencyNode dependencyNodeForDependency, ComparableVersion expectedVersion, ComparableVersion resolvedVersion) { 374 return context.getExclusions().stream().noneMatch(exclusion -> exclusion.matches(dependencyNodeForDependency, expectedVersion, resolvedVersion)); 375 } 376 377 /** 378 * Return a version object for an Artifact. 379 */ 380 private static ComparableVersion getVersion(DependencyNode dependencyNode) 381 throws OverConstrainedVersionException { 382 checkNotNull(dependencyNode, "dependencyNode is null"); 383 384 checkState(dependencyNode.getVersion() != null, "DependencyNode %s has a null version selected. Please report a bug!", dependencyNode); 385 return new ComparableVersion(dependencyNode.getVersion().toString()); 386 } 387}