001/* 002 * Licensed under the Apache License, Version 2.0 (the "License"); 003 * you may not use this file except in compliance with the License. 004 * You may obtain a copy of the License at 005 * 006 * http://www.apache.org/licenses/LICENSE-2.0 007 * 008 * Unless required by applicable law or agreed to in writing, software 009 * distributed under the License is distributed on an "AS IS" BASIS, 010 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 011 * See the License for the specific language governing permissions and 012 * limitations under the License. 013 */ 014 015package org.basepom.mojo.dvc.dependency; 016 017import static com.google.common.base.Preconditions.checkNotNull; 018import static com.google.common.base.Preconditions.checkState; 019import static com.google.common.collect.ImmutableList.toImmutableList; 020import static com.google.common.collect.ImmutableSet.toImmutableSet; 021import static java.lang.String.format; 022import static org.basepom.mojo.dvc.dependency.DependencyMapBuilder.convertToPomArtifact; 023 024import org.basepom.mojo.dvc.CheckExclusionsFilter; 025import org.basepom.mojo.dvc.Context; 026import org.basepom.mojo.dvc.PluginLog; 027import org.basepom.mojo.dvc.QualifiedName; 028import org.basepom.mojo.dvc.ScopeLimitingFilter; 029import org.basepom.mojo.dvc.strategy.Strategy; 030import org.basepom.mojo.dvc.version.VersionResolution; 031import org.basepom.mojo.dvc.version.VersionResolutionCollection; 032 033import java.time.Duration; 034import java.util.Collection; 035import java.util.Set; 036import java.util.concurrent.Callable; 037import java.util.concurrent.ExecutionException; 038import java.util.concurrent.Executors; 039 040import com.google.common.base.Joiner; 041import com.google.common.collect.ImmutableList; 042import com.google.common.collect.ImmutableSet; 043import com.google.common.collect.ImmutableSetMultimap; 044import com.google.common.util.concurrent.ListenableFuture; 045import com.google.common.util.concurrent.ListeningExecutorService; 046import com.google.common.util.concurrent.MoreExecutors; 047import com.google.common.util.concurrent.ThreadFactoryBuilder; 048import org.apache.maven.RepositoryUtils; 049import org.apache.maven.artifact.resolver.AbstractArtifactResolutionException; 050import org.apache.maven.artifact.versioning.ComparableVersion; 051import org.apache.maven.artifact.versioning.OverConstrainedVersionException; 052import org.apache.maven.plugin.MojoExecutionException; 053import org.apache.maven.project.DependencyResolutionException; 054import org.apache.maven.project.MavenProject; 055import org.apache.maven.project.ProjectBuildingException; 056import org.eclipse.aether.RepositorySystem; 057import org.eclipse.aether.artifact.Artifact; 058import org.eclipse.aether.artifact.ArtifactTypeRegistry; 059import org.eclipse.aether.graph.Dependency; 060import org.eclipse.aether.graph.DependencyFilter; 061import org.eclipse.aether.graph.DependencyNode; 062import org.eclipse.aether.resolution.VersionRangeRequest; 063import org.eclipse.aether.resolution.VersionRangeResolutionException; 064import org.eclipse.aether.resolution.VersionRangeResult; 065import org.eclipse.aether.util.filter.AndDependencyFilter; 066 067public final class DependencyTreeResolver 068 implements AutoCloseable { 069 070 private static final PluginLog LOG = new PluginLog(DependencyTreeResolver.class); 071 072 private static final int DEPENDENCY_RESOLUTION_NUM_THREADS = Runtime.getRuntime().availableProcessors() * 5; 073 074 private final Context context; 075 private final DependencyMap rootDependencyMap; 076 077 private final ListeningExecutorService executorService = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(DEPENDENCY_RESOLUTION_NUM_THREADS, 078 new ThreadFactoryBuilder().setNameFormat("dependency-version-check-worker-%s").setDaemon(true).build())); 079 080 public DependencyTreeResolver(final Context context, final DependencyMap rootDependencyMap) { 081 this.context = checkNotNull(context, "context is null"); 082 this.rootDependencyMap = checkNotNull(rootDependencyMap, "rootDependencyMap is null"); 083 } 084 085 @Override 086 public void close() { 087 MoreExecutors.shutdownAndAwaitTermination(executorService, Duration.ofSeconds(2)); 088 } 089 090 /** 091 * Creates a map of all dependency version resolutions used in this project in a given scope. The result is a map from names to a list of version numbers 092 * used in the project, based on the element requesting the version. 093 * <p> 094 * If the special scope "null" is used, a superset of all scopes is used (this is used by the check mojo). 095 * 096 * @param project The maven project to resolve all dependencies for. 097 * @param scopeFilter Limits the scopes to resolve. 098 * @return Map from qualified names to possible version resolutions. 099 * @throws MojoExecutionException Parallel dependency resolution failed. 100 */ 101 public ImmutableSetMultimap<QualifiedName, VersionResolutionCollection> computeResolutionMap(final MavenProject project, 102 final ScopeLimitingFilter scopeFilter) 103 throws MojoExecutionException { 104 checkNotNull(project, "project is null"); 105 checkNotNull(scopeFilter, "scope is null"); 106 107 final ImmutableSetMultimap.Builder<QualifiedName, VersionResolution> collector = ImmutableSetMultimap.builder(); 108 final ImmutableList.Builder<ListenableFuture<?>> futureBuilder = ImmutableList.builder(); 109 110 boolean useParallelDependencyResolution = context.useFastResolution(); 111 // Map from dependency name --> list of resolutions found on the tree 112 LOG.debug("Using parallel dependency resolution: %s", useParallelDependencyResolution); 113 114 final ImmutableList<Dependency> dependencies; 115 if (context.useDeepScan()) { 116 LOG.debug("Running deep scan"); 117 dependencies = ImmutableList.copyOf( 118 rootDependencyMap.getAllDependencies().values().stream().map(DependencyNode::getDependency).collect(toImmutableList())); 119 } else { 120 final ArtifactTypeRegistry stereotypes = context.getRepositorySystemSession().getArtifactTypeRegistry(); 121 122 dependencies = ImmutableList.copyOf( 123 project.getDependencies().stream().map(d -> RepositoryUtils.toDependency(d, stereotypes)).collect(toImmutableList())); 124 } 125 126 final ImmutableSet.Builder<Throwable> throwableBuilder = ImmutableSet.builder(); 127 128 if (useParallelDependencyResolution) { 129 for (final Dependency dependency : dependencies) { 130 futureBuilder.add(executorService.submit((Callable<Void>) () -> { 131 resolveProjectDependency(dependency, scopeFilter, collector); 132 return null; 133 })); 134 } 135 136 final ImmutableList<ListenableFuture<?>> futures = futureBuilder.build(); 137 138 for (final ListenableFuture<?> future : futures) { 139 try { 140 future.get(); 141 } catch (InterruptedException e) { 142 Thread.currentThread().interrupt(); 143 } catch (ExecutionException e) { 144 throwableBuilder.add(e.getCause()); 145 } 146 } 147 } else { 148 for (final Dependency dependency : dependencies) { 149 try { 150 resolveProjectDependency(dependency, scopeFilter, collector); 151 } catch (Throwable t) { 152 throwableBuilder.add(t); 153 } 154 } 155 } 156 157 final Set<Throwable> throwables = throwableBuilder.build(); 158 if (!throwables.isEmpty()) { 159 throw processResolveProjectDependencyException(throwableBuilder.build()); 160 } 161 162 return VersionResolutionCollection.toResolutionMap(collector.build()); 163 } 164 165 private static MojoExecutionException processResolveProjectDependencyException(Set<Throwable> throwables) { 166 ImmutableSet.Builder<String> failedDependenciesBuilder = ImmutableSet.builder(); 167 ImmutableSet.Builder<String> messageBuilder = ImmutableSet.builder(); 168 for (Throwable t : throwables) { 169 if (t instanceof DependencyResolutionException) { 170 ((DependencyResolutionException) t).getResult().getUnresolvedDependencies() 171 .forEach(d -> failedDependenciesBuilder.add(printDependency(d))); 172 } else { 173 messageBuilder.add(t.getMessage()); 174 } 175 } 176 177 String message = Joiner.on(" \n").join(messageBuilder.build()); 178 Set<String> failedDependencies = failedDependenciesBuilder.build(); 179 if (!failedDependencies.isEmpty()) { 180 if (!message.isEmpty()) { 181 message += "\n"; 182 } 183 message += "Could not resolve dependencies: [" + Joiner.on(", ").join(failedDependencies) + "]"; 184 } 185 return new MojoExecutionException(message); 186 } 187 188 private static String printDependency(Dependency d) { 189 return d.getArtifact() + " [" + d.getScope() + (d.isOptional() ? ", optional" : "") + "]"; 190 } 191 192 /** 193 * Called for any direct project dependency. Factored out from {@link #computeResolutionMap} to allow parallel evaluation of dependencies to speed up the 194 * process. 195 */ 196 private void resolveProjectDependency(final Dependency dependency, 197 final ScopeLimitingFilter visibleScopes, 198 final ImmutableSetMultimap.Builder<QualifiedName, VersionResolution> collector) 199 throws MojoExecutionException, DependencyResolutionException, AbstractArtifactResolutionException, VersionRangeResolutionException { 200 final QualifiedName dependencyName = QualifiedName.fromDependency(dependency); 201 202 // see if the resolved, direct dependency contain this name. 203 // If not, the dependency is declared in a scope that is not used (it was filtered out by the scope filter 204 // when the map was created. 205 if (rootDependencyMap.getDirectDependencies().containsKey(dependencyName)) { 206 // a direct dependency 207 final DependencyNode projectDependencyNode = rootDependencyMap.getDirectDependencies().get(dependencyName); 208 assert projectDependencyNode != null; 209 210 checkState(visibleScopes.accept(projectDependencyNode, ImmutableList.of()), 211 "Dependency %s maps to %s, but scope filter would exclude it. This should never happen!", dependency, projectDependencyNode); 212 computeVersionResolutionForDirectDependency(collector, dependency, projectDependencyNode); 213 } 214 215 // could be a dependency in the full dependency list 216 final DependencyNode projectDependencyNode = rootDependencyMap.getAllDependencies().get(dependencyName); 217 // A project dependency could be e.g. in test scope, but the map has been computed in a narrower scope (e.g. compile) 218 // in that case, it does not contain a dependency node for the dependency. That is ok, simply ignore it. 219 if (projectDependencyNode == null) { 220 return; 221 } 222 checkState(visibleScopes.accept(projectDependencyNode, ImmutableList.of()), 223 "Dependency %s maps to %s, but scope filter would exclude it. This should never happen!", dependency, projectDependencyNode); 224 225 try { 226 // remove the test scope for resolving all the transitive dependencies. Anything that was pulled in in test scope, 227 // now needs its dependencies resolved in compile+runtime scope, not test scope. 228 final ScopeLimitingFilter dependencyScope = ScopeLimitingFilter.computeTransitiveScope(dependency.getScope()); 229 computeVersionResolutionForTransitiveDependencies(collector, dependency, projectDependencyNode, dependencyScope); 230 } catch (ProjectBuildingException e) { 231 // This is an optimization and a bug workaround at the same time. Some artifacts exist that 232 // specify a packaging that is not natively supported by maven (e.g. bundle of OSGi bundles), however they 233 // do not bring the necessary extensions to deal with that type. As a result, this causes a "could not read model" 234 // exception. Ignore the transitive dependencies if the project node does not suggest any child artifacts. 235 if (projectDependencyNode.getChildren().isEmpty()) { 236 LOG.debug("Ignoring model building exception for %s, no children were declared", dependency); 237 } else { 238 LOG.warn("Could not read POM for %s, ignoring project and its dependencies!", dependency); 239 } 240 } 241 } 242 243 /** 244 * Create a version resolution for the given direct requestingDependency and artifact. 245 */ 246 private void computeVersionResolutionForDirectDependency( 247 final ImmutableSetMultimap.Builder<QualifiedName, VersionResolution> collector, 248 final Dependency requestingDependency, 249 final DependencyNode resolvedDependencyNode) 250 throws AbstractArtifactResolutionException, VersionRangeResolutionException, MojoExecutionException { 251 final QualifiedName requestingDependencyName = QualifiedName.fromDependency(requestingDependency); 252 253 final RepositorySystem repoSystem = context.getRepositorySystem(); 254 255 Artifact artifact = convertToPomArtifact(requestingDependency.getArtifact()); 256 if (artifact.isSnapshot()) { 257 // convert version of a snapshot artifact to be SNAPSHOT, otherwise the 258 // version range resolver will try to match the timestamp version 259 artifact = artifact.setVersion(artifact.getBaseVersion()); 260 } 261 262 final VersionRangeRequest request = context.createVersionRangeRequest(artifact); 263 final VersionRangeResult result = repoSystem.resolveVersionRange(context.getRepositorySystemSession(), request); 264 265 if (!result.getVersions().contains(resolvedDependencyNode.getVersion())) { 266 throw new MojoExecutionException( 267 format("Cannot determine the recommended version of dependency '%s'; its version specification is '%s', and the resolved version is '%s'.", 268 requestingDependency, requestingDependency.getArtifact().getBaseVersion(), resolvedDependencyNode.getVersion())); 269 } 270 271 // dependency range contains the project version (or matches it) 272 273 // version from the dependency artifact 274 final ComparableVersion expectedVersion = getVersion(resolvedDependencyNode); 275 276 // this is a direct dependency; it made it through the filter in resolveProjectDependency. 277 final boolean managedDependency = (resolvedDependencyNode.getManagedBits() & DependencyNode.MANAGED_VERSION) != 0; 278 final VersionResolution resolution = VersionResolution.forDirectDependency(QualifiedName.fromProject(context.getRootProject()), expectedVersion, 279 managedDependency); 280 281 if (isIncluded(resolvedDependencyNode, expectedVersion, expectedVersion)) { 282 final Strategy strategy = context.getStrategyCache().forQualifiedName(requestingDependencyName); 283 checkState(strategy != null, "Strategy for %s is null, this should never happen (could not find default strategy?", requestingDependencyName); 284 285 if (!strategy.isCompatible(expectedVersion, expectedVersion)) { 286 resolution.conflict(); 287 } 288 } else { 289 LOG.debug("VersionResolution %s is excluded by configuration.", resolution); 290 } 291 292 synchronized (collector) { 293 collector.put(requestingDependencyName, resolution); 294 } 295 } 296 297 /** 298 * Resolve all transitive dependencies relative to a given dependency, based off the artifact given. A scope filter can be added which limits the results to 299 * the scopes present in that filter. 300 */ 301 private void computeVersionResolutionForTransitiveDependencies( 302 final ImmutableSetMultimap.Builder<QualifiedName, VersionResolution> collector, 303 final Dependency requestingDependency, 304 final DependencyNode dependencyNodeForDependency, 305 final DependencyFilter scopeFilter) 306 throws AbstractArtifactResolutionException, ProjectBuildingException, DependencyResolutionException { 307 final AndDependencyFilter filter = new AndDependencyFilter(scopeFilter, new CheckExclusionsFilter(requestingDependency.getExclusions())); 308 309 final DependencyMap dependencyMap = new DependencyMapBuilder(context).mapDependency(dependencyNodeForDependency, filter); 310 final Collection<DependencyNode> transitiveDependencies = dependencyMap.getAllDependencies().values(); 311 final QualifiedName requestingDependencyName = QualifiedName.fromDependency(requestingDependency); 312 313 final ImmutableSet<DependencyNode> filteredDependencies = transitiveDependencies.stream() 314 .filter(d -> scopeFilter.accept(d, ImmutableList.of())) 315 .filter(d -> !d.getDependency().isOptional()) 316 .collect(toImmutableSet()); 317 318 for (final DependencyNode dependencyNode : filteredDependencies) { 319 final QualifiedName dependencyName = QualifiedName.fromDependencyNode(dependencyNode); 320 321 final DependencyNode projectDependencyNode = rootDependencyMap.getAllDependencies().get(dependencyName); 322 if (projectDependencyNode == null) { 323 // the next condition can happen if a dependency is required by one dependency but then overridden by another. e.g. 324 // 325 // guava (*29.1-jre*, 25.1-android) 326 // guava 25.1-android depends on org.checkerframework:checker-compat-qual 327 // guava 29.1-jre depends on org.checkerframework:checker-qual 328 // 329 // as the dependency resolver chose 29.1-jre, only the "checker-qual" dependency will show on the final classpath 330 // however, when resolving all dependencies, the code will resolve the dependency which pulls in guava-25.1-android. 331 // For that dependency, there will be "checker-compat-qual" in the list of dependencies, but when the code tries to 332 // resolve the actual classpath dependency, the "checker-compat-qual" dependency is not in the final classpath. 333 // 334 // This is normal situation and the dependency can just be dropped. 335 // 336 continue; 337 } 338 339 final ComparableVersion resolvedVersion = getVersion(projectDependencyNode); 340 final ComparableVersion expectedVersion = getVersion(dependencyNode); 341 342 final boolean managedDependency = (projectDependencyNode.getManagedBits() & DependencyNode.MANAGED_VERSION) != 0; 343 final VersionResolution resolution = VersionResolution.forTransitiveDependency(requestingDependencyName, expectedVersion, managedDependency); 344 345 if (isIncluded(dependencyNode, expectedVersion, resolvedVersion)) { 346 final Strategy strategy = context.getStrategyCache().forQualifiedName(dependencyName); 347 checkState(strategy != null, "Strategy for %s is null, this should never happen (could not find default strategy?", dependencyName); 348 349 if (!strategy.isCompatible(expectedVersion, resolvedVersion)) { 350 resolution.conflict(); 351 } 352 } 353 synchronized (collector) { 354 collector.put(dependencyName, resolution); 355 } 356 } 357 } 358 359 /** 360 * Returns true if a given artifact and version should be checked. 361 */ 362 private boolean isIncluded(DependencyNode dependencyNodeForDependency, ComparableVersion expectedVersion, ComparableVersion resolvedVersion) { 363 return context.getExclusions().stream().noneMatch(exclusion -> exclusion.matches(dependencyNodeForDependency, expectedVersion, resolvedVersion)); 364 } 365 366 /** 367 * Return a version object for an Artifact. 368 */ 369 private static ComparableVersion getVersion(DependencyNode dependencyNode) 370 throws OverConstrainedVersionException { 371 checkNotNull(dependencyNode, "dependencyNode is null"); 372 373 checkState(dependencyNode.getVersion() != null, "DependencyNode %s has a null version selected. Please report a bug!", dependencyNode); 374 return new ComparableVersion(dependencyNode.getVersion().toString()); 375 } 376}