001/*
002 * Licensed under the Apache License, Version 2.0 (the "License");
003 * you may not use this file except in compliance with the License.
004 * You may obtain a copy of the License at
005 *
006 * http://www.apache.org/licenses/LICENSE-2.0
007 *
008 * Unless required by applicable law or agreed to in writing, software
009 * distributed under the License is distributed on an "AS IS" BASIS,
010 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
011 * See the License for the specific language governing permissions and
012 * limitations under the License.
013 */
014
015package org.basepom.mojo.dvc.dependency;
016
017import static com.google.common.base.Preconditions.checkNotNull;
018import static com.google.common.base.Preconditions.checkState;
019import static com.google.common.collect.ImmutableList.toImmutableList;
020import static com.google.common.collect.ImmutableSet.toImmutableSet;
021import static java.lang.String.format;
022import static org.basepom.mojo.dvc.dependency.DependencyMapBuilder.convertToPomArtifact;
023
024import org.basepom.mojo.dvc.CheckExclusionsFilter;
025import org.basepom.mojo.dvc.Context;
026import org.basepom.mojo.dvc.PluginLog;
027import org.basepom.mojo.dvc.QualifiedName;
028import org.basepom.mojo.dvc.ScopeLimitingFilter;
029import org.basepom.mojo.dvc.strategy.Strategy;
030import org.basepom.mojo.dvc.version.VersionResolution;
031import org.basepom.mojo.dvc.version.VersionResolutionCollection;
032
033import java.time.Duration;
034import java.util.Collection;
035import java.util.Set;
036import java.util.concurrent.Callable;
037import java.util.concurrent.ExecutionException;
038import java.util.concurrent.Executors;
039
040import com.google.common.base.Joiner;
041import com.google.common.collect.ImmutableList;
042import com.google.common.collect.ImmutableSet;
043import com.google.common.collect.ImmutableSetMultimap;
044import com.google.common.util.concurrent.ListenableFuture;
045import com.google.common.util.concurrent.ListeningExecutorService;
046import com.google.common.util.concurrent.MoreExecutors;
047import com.google.common.util.concurrent.ThreadFactoryBuilder;
048import org.apache.maven.RepositoryUtils;
049import org.apache.maven.artifact.resolver.AbstractArtifactResolutionException;
050import org.apache.maven.artifact.versioning.ComparableVersion;
051import org.apache.maven.artifact.versioning.OverConstrainedVersionException;
052import org.apache.maven.plugin.MojoExecutionException;
053import org.apache.maven.project.DependencyResolutionException;
054import org.apache.maven.project.MavenProject;
055import org.apache.maven.project.ProjectBuildingException;
056import org.eclipse.aether.RepositorySystem;
057import org.eclipse.aether.artifact.Artifact;
058import org.eclipse.aether.artifact.ArtifactTypeRegistry;
059import org.eclipse.aether.graph.Dependency;
060import org.eclipse.aether.graph.DependencyFilter;
061import org.eclipse.aether.graph.DependencyNode;
062import org.eclipse.aether.resolution.VersionRangeRequest;
063import org.eclipse.aether.resolution.VersionRangeResolutionException;
064import org.eclipse.aether.resolution.VersionRangeResult;
065import org.eclipse.aether.util.filter.AndDependencyFilter;
066
067public final class DependencyTreeResolver
068        implements AutoCloseable {
069
070    private static final PluginLog LOG = new PluginLog(DependencyTreeResolver.class);
071
072    private static final int DEPENDENCY_RESOLUTION_NUM_THREADS = Runtime.getRuntime().availableProcessors() * 5;
073
074    private final Context context;
075    private final DependencyMap rootDependencyMap;
076
077    private final ListeningExecutorService executorService = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(DEPENDENCY_RESOLUTION_NUM_THREADS,
078            new ThreadFactoryBuilder().setNameFormat("dependency-version-check-worker-%s").setDaemon(true).build()));
079
080    public DependencyTreeResolver(final Context context, final DependencyMap rootDependencyMap) {
081        this.context = checkNotNull(context, "context is null");
082        this.rootDependencyMap = checkNotNull(rootDependencyMap, "rootDependencyMap is null");
083    }
084
085    @Override
086    public void close() {
087        MoreExecutors.shutdownAndAwaitTermination(executorService, Duration.ofSeconds(2));
088    }
089
090    /**
091     * Creates a map of all dependency version resolutions used in this project in a given scope. The result is a map from names to a list of version numbers
092     * used in the project, based on the element requesting the version.
093     * <p>
094     * If the special scope "null" is used, a superset of all scopes is used (this is used by the check mojo).
095     *
096     * @param project     The maven project to resolve all dependencies for.
097     * @param scopeFilter Limits the scopes to resolve.
098     * @return Map from qualified names to possible version resolutions.
099     * @throws MojoExecutionException Parallel dependency resolution failed.
100     */
101    public ImmutableSetMultimap<QualifiedName, VersionResolutionCollection> computeResolutionMap(final MavenProject project,
102            final ScopeLimitingFilter scopeFilter)
103            throws MojoExecutionException {
104        checkNotNull(project, "project is null");
105        checkNotNull(scopeFilter, "scope is null");
106
107        final ImmutableSetMultimap.Builder<QualifiedName, VersionResolution> collector = ImmutableSetMultimap.builder();
108        final ImmutableList.Builder<ListenableFuture<?>> futureBuilder = ImmutableList.builder();
109
110        boolean useParallelDependencyResolution = context.useFastResolution();
111        // Map from dependency name --> list of resolutions found on the tree
112        LOG.debug("Using parallel dependency resolution: %s", useParallelDependencyResolution);
113
114        final ImmutableList<Dependency> dependencies;
115        if (context.useDeepScan()) {
116            LOG.debug("Running deep scan");
117            dependencies = ImmutableList.copyOf(
118                    rootDependencyMap.getAllDependencies().values().stream().map(DependencyNode::getDependency).collect(toImmutableList()));
119        } else {
120            final ArtifactTypeRegistry stereotypes = context.getRepositorySystemSession().getArtifactTypeRegistry();
121
122            dependencies = ImmutableList.copyOf(
123                    project.getDependencies().stream().map(d -> RepositoryUtils.toDependency(d, stereotypes)).collect(toImmutableList()));
124        }
125
126        final ImmutableSet.Builder<Throwable> throwableBuilder = ImmutableSet.builder();
127
128        if (useParallelDependencyResolution) {
129            for (final Dependency dependency : dependencies) {
130                futureBuilder.add(executorService.submit((Callable<Void>) () -> {
131                    resolveProjectDependency(dependency, scopeFilter, collector);
132                    return null;
133                }));
134            }
135
136            final ImmutableList<ListenableFuture<?>> futures = futureBuilder.build();
137
138            for (final ListenableFuture<?> future : futures) {
139                try {
140                    future.get();
141                } catch (InterruptedException e) {
142                    Thread.currentThread().interrupt();
143                } catch (ExecutionException e) {
144                    throwableBuilder.add(e.getCause());
145                }
146            }
147        } else {
148            for (final Dependency dependency : dependencies) {
149                try {
150                    resolveProjectDependency(dependency, scopeFilter, collector);
151                } catch (Throwable t) {
152                    throwableBuilder.add(t);
153                }
154            }
155        }
156
157        final Set<Throwable> throwables = throwableBuilder.build();
158        if (!throwables.isEmpty()) {
159            throw processResolveProjectDependencyException(throwableBuilder.build());
160        }
161
162        return VersionResolutionCollection.toResolutionMap(collector.build());
163    }
164
165    private static MojoExecutionException processResolveProjectDependencyException(Set<Throwable> throwables) {
166        ImmutableSet.Builder<String> failedDependenciesBuilder = ImmutableSet.builder();
167        ImmutableSet.Builder<String> messageBuilder = ImmutableSet.builder();
168        for (Throwable t : throwables) {
169            if (t instanceof DependencyResolutionException) {
170                ((DependencyResolutionException) t).getResult().getUnresolvedDependencies()
171                        .forEach(d -> failedDependenciesBuilder.add(printDependency(d)));
172            } else {
173                messageBuilder.add(t.getMessage());
174            }
175        }
176
177        String message = Joiner.on("    \n").join(messageBuilder.build());
178        Set<String> failedDependencies = failedDependenciesBuilder.build();
179        if (!failedDependencies.isEmpty()) {
180            if (!message.isEmpty()) {
181                message += "\n";
182            }
183            message += "Could not resolve dependencies: [" + Joiner.on(", ").join(failedDependencies) + "]";
184        }
185        return new MojoExecutionException(message);
186    }
187
188    private static String printDependency(Dependency d) {
189        return d.getArtifact() + " [" + d.getScope() + (d.isOptional() ? ", optional" : "") + "]";
190    }
191
192    /**
193     * Called for any direct project dependency. Factored out from {@link #computeResolutionMap} to allow parallel evaluation of dependencies to speed up the
194     * process.
195     */
196    private void resolveProjectDependency(final Dependency dependency,
197            final ScopeLimitingFilter visibleScopes,
198            final ImmutableSetMultimap.Builder<QualifiedName, VersionResolution> collector)
199            throws MojoExecutionException, DependencyResolutionException, AbstractArtifactResolutionException, VersionRangeResolutionException {
200        final QualifiedName dependencyName = QualifiedName.fromDependency(dependency);
201
202        // see if the resolved, direct dependency contain this name.
203        // If not, the dependency is declared in a scope that is not used (it was filtered out by the scope filter
204        // when the map was created.
205        if (rootDependencyMap.getDirectDependencies().containsKey(dependencyName)) {
206            // a direct dependency
207            final DependencyNode projectDependencyNode = rootDependencyMap.getDirectDependencies().get(dependencyName);
208            assert projectDependencyNode != null;
209
210            checkState(visibleScopes.accept(projectDependencyNode, ImmutableList.of()),
211                    "Dependency %s maps to %s, but scope filter would exclude it. This should never happen!", dependency, projectDependencyNode);
212            computeVersionResolutionForDirectDependency(collector, dependency, projectDependencyNode);
213        }
214
215        // could be a dependency in the full dependency list
216        final DependencyNode projectDependencyNode = rootDependencyMap.getAllDependencies().get(dependencyName);
217        // A project dependency could be e.g. in test scope, but the map has been computed in a narrower scope (e.g. compile)
218        // in that case, it does not contain a dependency node for the dependency. That is ok, simply ignore it.
219        if (projectDependencyNode == null) {
220            return;
221        }
222        checkState(visibleScopes.accept(projectDependencyNode, ImmutableList.of()),
223                "Dependency %s maps to %s, but scope filter would exclude it. This should never happen!", dependency, projectDependencyNode);
224
225        try {
226            // remove the test scope for resolving all the transitive dependencies. Anything that was pulled in in test scope,
227            // now needs its dependencies resolved in compile+runtime scope, not test scope.
228            final ScopeLimitingFilter dependencyScope = ScopeLimitingFilter.computeTransitiveScope(dependency.getScope());
229            computeVersionResolutionForTransitiveDependencies(collector, dependency, projectDependencyNode, dependencyScope);
230        } catch (ProjectBuildingException e) {
231            // This is an optimization and a bug workaround at the same time. Some artifacts exist that
232            // specify a packaging that is not natively supported by maven (e.g. bundle of OSGi bundles), however they
233            // do not bring the necessary extensions to deal with that type. As a result, this causes a "could not read model"
234            // exception. Ignore the transitive dependencies if the project node does not suggest any child artifacts.
235            if (projectDependencyNode.getChildren().isEmpty()) {
236                LOG.debug("Ignoring model building exception for %s, no children were declared", dependency);
237            } else {
238                LOG.warn("Could not read POM for %s, ignoring project and its dependencies!", dependency);
239            }
240        }
241    }
242
243    /**
244     * Create a version resolution for the given direct requestingDependency and artifact.
245     */
246    private void computeVersionResolutionForDirectDependency(
247            final ImmutableSetMultimap.Builder<QualifiedName, VersionResolution> collector,
248            final Dependency requestingDependency,
249            final DependencyNode resolvedDependencyNode)
250            throws AbstractArtifactResolutionException, VersionRangeResolutionException, MojoExecutionException {
251        final QualifiedName requestingDependencyName = QualifiedName.fromDependency(requestingDependency);
252
253        final RepositorySystem repoSystem = context.getRepositorySystem();
254
255        Artifact artifact = convertToPomArtifact(requestingDependency.getArtifact());
256        if (artifact.isSnapshot()) {
257            // convert version of a snapshot artifact to be SNAPSHOT, otherwise the
258            // version range resolver will try to match the timestamp version
259            artifact = artifact.setVersion(artifact.getBaseVersion());
260        }
261
262        final VersionRangeRequest request = context.createVersionRangeRequest(artifact);
263        final VersionRangeResult result = repoSystem.resolveVersionRange(context.getRepositorySystemSession(), request);
264
265        if (!result.getVersions().contains(resolvedDependencyNode.getVersion())) {
266            throw new MojoExecutionException(
267                    format("Cannot determine the recommended version of dependency '%s'; its version specification is '%s', and the resolved version is '%s'.",
268                            requestingDependency, requestingDependency.getArtifact().getBaseVersion(), resolvedDependencyNode.getVersion()));
269        }
270
271        // dependency range contains the project version (or matches it)
272
273        // version from the dependency artifact
274        final ComparableVersion expectedVersion = getVersion(resolvedDependencyNode);
275
276        // this is a direct dependency; it made it through the filter in resolveProjectDependency.
277        final boolean managedDependency = (resolvedDependencyNode.getManagedBits() & DependencyNode.MANAGED_VERSION) != 0;
278        final VersionResolution resolution = VersionResolution.forDirectDependency(QualifiedName.fromProject(context.getRootProject()), expectedVersion,
279                managedDependency);
280
281        if (isIncluded(resolvedDependencyNode, expectedVersion, expectedVersion)) {
282            final Strategy strategy = context.getStrategyCache().forQualifiedName(requestingDependencyName);
283            checkState(strategy != null, "Strategy for %s is null, this should never happen (could not find default strategy?", requestingDependencyName);
284
285            if (!strategy.isCompatible(expectedVersion, expectedVersion)) {
286                resolution.conflict();
287            }
288        } else {
289            LOG.debug("VersionResolution %s is excluded by configuration.", resolution);
290        }
291
292        synchronized (collector) {
293            collector.put(requestingDependencyName, resolution);
294        }
295    }
296
297    /**
298     * Resolve all transitive dependencies relative to a given dependency, based off the artifact given. A scope filter can be added which limits the results to
299     * the scopes present in that filter.
300     */
301    private void computeVersionResolutionForTransitiveDependencies(
302            final ImmutableSetMultimap.Builder<QualifiedName, VersionResolution> collector,
303            final Dependency requestingDependency,
304            final DependencyNode dependencyNodeForDependency,
305            final DependencyFilter scopeFilter)
306            throws AbstractArtifactResolutionException, ProjectBuildingException, DependencyResolutionException {
307        final AndDependencyFilter filter = new AndDependencyFilter(scopeFilter, new CheckExclusionsFilter(requestingDependency.getExclusions()));
308
309        final DependencyMap dependencyMap = new DependencyMapBuilder(context).mapDependency(dependencyNodeForDependency, filter);
310        final Collection<DependencyNode> transitiveDependencies = dependencyMap.getAllDependencies().values();
311        final QualifiedName requestingDependencyName = QualifiedName.fromDependency(requestingDependency);
312
313        final ImmutableSet<DependencyNode> filteredDependencies = transitiveDependencies.stream()
314                .filter(d -> scopeFilter.accept(d, ImmutableList.of()))
315                .filter(d -> !d.getDependency().isOptional())
316                .collect(toImmutableSet());
317
318        for (final DependencyNode dependencyNode : filteredDependencies) {
319            final QualifiedName dependencyName = QualifiedName.fromDependencyNode(dependencyNode);
320
321            final DependencyNode projectDependencyNode = rootDependencyMap.getAllDependencies().get(dependencyName);
322            if (projectDependencyNode == null) {
323                // the next condition can happen if a dependency is required by one dependency but then overridden by another. e.g.
324                //
325                //   guava (*29.1-jre*, 25.1-android)
326                //    guava 25.1-android depends on org.checkerframework:checker-compat-qual
327                //    guava 29.1-jre depends on org.checkerframework:checker-qual
328                //
329                // as the dependency resolver chose 29.1-jre, only the "checker-qual" dependency will show on the final classpath
330                // however, when resolving all dependencies, the code will resolve the dependency which pulls in guava-25.1-android.
331                // For that dependency, there will be "checker-compat-qual" in the list of dependencies, but when the code tries to
332                // resolve the actual classpath dependency, the "checker-compat-qual" dependency is not in the final classpath.
333                //
334                // This is normal situation and the dependency can just be dropped.
335                //
336                continue;
337            }
338
339            final ComparableVersion resolvedVersion = getVersion(projectDependencyNode);
340            final ComparableVersion expectedVersion = getVersion(dependencyNode);
341
342            final boolean managedDependency = (projectDependencyNode.getManagedBits() & DependencyNode.MANAGED_VERSION) != 0;
343            final VersionResolution resolution = VersionResolution.forTransitiveDependency(requestingDependencyName, expectedVersion, managedDependency);
344
345            if (isIncluded(dependencyNode, expectedVersion, resolvedVersion)) {
346                final Strategy strategy = context.getStrategyCache().forQualifiedName(dependencyName);
347                checkState(strategy != null, "Strategy for %s is null, this should never happen (could not find default strategy?", dependencyName);
348
349                if (!strategy.isCompatible(expectedVersion, resolvedVersion)) {
350                    resolution.conflict();
351                }
352            }
353            synchronized (collector) {
354                collector.put(dependencyName, resolution);
355            }
356        }
357    }
358
359    /**
360     * Returns true if a given artifact and version should be checked.
361     */
362    private boolean isIncluded(DependencyNode dependencyNodeForDependency, ComparableVersion expectedVersion, ComparableVersion resolvedVersion) {
363        return context.getExclusions().stream().noneMatch(exclusion -> exclusion.matches(dependencyNodeForDependency, expectedVersion, resolvedVersion));
364    }
365
366    /**
367     * Return a version object for an Artifact.
368     */
369    private static ComparableVersion getVersion(DependencyNode dependencyNode)
370            throws OverConstrainedVersionException {
371        checkNotNull(dependencyNode, "dependencyNode is null");
372
373        checkState(dependencyNode.getVersion() != null, "DependencyNode %s has a null version selected. Please report a bug!", dependencyNode);
374        return new ComparableVersion(dependencyNode.getVersion().toString());
375    }
376}