001/*
002 * Licensed under the Apache License, Version 2.0 (the "License");
003 * you may not use this file except in compliance with the License.
004 * You may obtain a copy of the License at
005 *
006 * http://www.apache.org/licenses/LICENSE-2.0
007 *
008 * Unless required by applicable law or agreed to in writing, software
009 * distributed under the License is distributed on an "AS IS" BASIS,
010 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
011 * See the License for the specific language governing permissions and
012 * limitations under the License.
013 */
014
015package org.basepom.mojo.dvc.dependency;
016
017import static com.google.common.base.Preconditions.checkNotNull;
018import static com.google.common.base.Preconditions.checkState;
019import static com.google.common.collect.ImmutableList.toImmutableList;
020import static com.google.common.collect.ImmutableSet.toImmutableSet;
021import static java.lang.String.format;
022import static org.basepom.mojo.dvc.dependency.DependencyMapBuilder.convertToPomArtifact;
023
024import org.basepom.mojo.dvc.CheckExclusionsFilter;
025import org.basepom.mojo.dvc.Context;
026import org.basepom.mojo.dvc.PluginLog;
027import org.basepom.mojo.dvc.QualifiedName;
028import org.basepom.mojo.dvc.ScopeLimitingFilter;
029import org.basepom.mojo.dvc.strategy.Strategy;
030import org.basepom.mojo.dvc.version.VersionResolution;
031import org.basepom.mojo.dvc.version.VersionResolutionCollection;
032
033import java.time.Duration;
034import java.util.Collection;
035import java.util.Set;
036import java.util.concurrent.Callable;
037import java.util.concurrent.ExecutionException;
038import java.util.concurrent.Executors;
039import java.util.concurrent.locks.Lock;
040import java.util.concurrent.locks.ReentrantLock;
041
042import com.google.common.base.Joiner;
043import com.google.common.collect.ImmutableList;
044import com.google.common.collect.ImmutableSet;
045import com.google.common.collect.ImmutableSetMultimap;
046import com.google.common.util.concurrent.ListenableFuture;
047import com.google.common.util.concurrent.ListeningExecutorService;
048import com.google.common.util.concurrent.MoreExecutors;
049import com.google.common.util.concurrent.ThreadFactoryBuilder;
050import org.apache.maven.RepositoryUtils;
051import org.apache.maven.artifact.resolver.AbstractArtifactResolutionException;
052import org.apache.maven.artifact.versioning.ComparableVersion;
053import org.apache.maven.artifact.versioning.OverConstrainedVersionException;
054import org.apache.maven.plugin.MojoExecutionException;
055import org.apache.maven.project.DependencyResolutionException;
056import org.apache.maven.project.MavenProject;
057import org.apache.maven.project.ProjectBuildingException;
058import org.eclipse.aether.RepositorySystem;
059import org.eclipse.aether.artifact.Artifact;
060import org.eclipse.aether.artifact.ArtifactTypeRegistry;
061import org.eclipse.aether.graph.Dependency;
062import org.eclipse.aether.graph.DependencyFilter;
063import org.eclipse.aether.graph.DependencyNode;
064import org.eclipse.aether.resolution.VersionRangeRequest;
065import org.eclipse.aether.resolution.VersionRangeResolutionException;
066import org.eclipse.aether.resolution.VersionRangeResult;
067import org.eclipse.aether.util.filter.AndDependencyFilter;
068
069public final class DependencyTreeResolver
070        implements AutoCloseable {
071
072    private static final PluginLog LOG = new PluginLog(DependencyTreeResolver.class);
073
074    private static final int DEPENDENCY_RESOLUTION_NUM_THREADS = Runtime.getRuntime().availableProcessors() * 5;
075
076    private final Lock collectorLock = new ReentrantLock();
077
078    private final Context context;
079    private final DependencyMap rootDependencyMap;
080
081    private final ListeningExecutorService executorService = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(DEPENDENCY_RESOLUTION_NUM_THREADS,
082            new ThreadFactoryBuilder().setNameFormat("dependency-version-check-worker-%s").setDaemon(true).build()));
083
084    public DependencyTreeResolver(final Context context, final DependencyMap rootDependencyMap) {
085        this.context = checkNotNull(context, "context is null");
086        this.rootDependencyMap = checkNotNull(rootDependencyMap, "rootDependencyMap is null");
087    }
088
089    @Override
090    public void close() {
091        MoreExecutors.shutdownAndAwaitTermination(executorService, Duration.ofSeconds(2));
092    }
093
094    /**
095     * Creates a map of all dependency version resolutions used in this project in a given scope. The result is a map from names to a list of version numbers
096     * used in the project, based on the element requesting the version.
097     * <p>
098     * If the special scope "null" is used, a superset of all scopes is used (this is used by the check mojo).
099     *
100     * @param project     The maven project to resolve all dependencies for.
101     * @param scopeFilter Limits the scopes to resolve.
102     * @return Map from qualified names to possible version resolutions.
103     * @throws MojoExecutionException Parallel dependency resolution failed.
104     */
105    public ImmutableSetMultimap<QualifiedName, VersionResolutionCollection> computeResolutionMap(final MavenProject project,
106            final ScopeLimitingFilter scopeFilter)
107            throws MojoExecutionException {
108        checkNotNull(project, "project is null");
109        checkNotNull(scopeFilter, "scope is null");
110
111        final ImmutableSetMultimap.Builder<QualifiedName, VersionResolution> collector = ImmutableSetMultimap.builder();
112        final ImmutableList.Builder<ListenableFuture<?>> futureBuilder = ImmutableList.builder();
113
114        boolean useParallelDependencyResolution = context.useFastResolution();
115        // Map from dependency name --> list of resolutions found on the tree
116        LOG.debug("Using parallel dependency resolution: %s", useParallelDependencyResolution);
117
118        final ImmutableList<Dependency> dependencies;
119        if (context.useDeepScan()) {
120            LOG.debug("Running deep scan");
121            dependencies = ImmutableList.copyOf(
122                    rootDependencyMap.getAllDependencies().values().stream().map(DependencyNode::getDependency).collect(toImmutableList()));
123        } else {
124            final ArtifactTypeRegistry stereotypes = context.getRepositorySystemSession().getArtifactTypeRegistry();
125
126            dependencies = ImmutableList.copyOf(
127                    project.getDependencies().stream().map(d -> RepositoryUtils.toDependency(d, stereotypes)).collect(toImmutableList()));
128        }
129
130        final ImmutableSet.Builder<Throwable> throwableBuilder = ImmutableSet.builder();
131
132        if (useParallelDependencyResolution) {
133            for (final Dependency dependency : dependencies) {
134                futureBuilder.add(executorService.submit((Callable<Void>) () -> {
135                    resolveProjectDependency(dependency, scopeFilter, collector);
136                    return null;
137                }));
138            }
139
140            final ImmutableList<ListenableFuture<?>> futures = futureBuilder.build();
141
142            for (final ListenableFuture<?> future : futures) {
143                try {
144                    future.get();
145                } catch (InterruptedException e) {
146                    Thread.currentThread().interrupt();
147                } catch (ExecutionException e) {
148                    throwableBuilder.add(e.getCause());
149                }
150            }
151        } else {
152            for (final Dependency dependency : dependencies) {
153                try {
154                    resolveProjectDependency(dependency, scopeFilter, collector);
155                } catch (Exception e) {
156                    throwableBuilder.add(e);
157                }
158            }
159        }
160
161        final Set<Throwable> throwables = throwableBuilder.build();
162        if (!throwables.isEmpty()) {
163            throw processResolveProjectDependencyException(throwableBuilder.build());
164        }
165
166        return VersionResolutionCollection.toResolutionMap(collector.build());
167    }
168
169    private static MojoExecutionException processResolveProjectDependencyException(Set<Throwable> throwables) {
170        ImmutableSet.Builder<String> failedDependenciesBuilder = ImmutableSet.builder();
171        ImmutableSet.Builder<String> messageBuilder = ImmutableSet.builder();
172        for (Throwable t : throwables) {
173            if (t instanceof DependencyResolutionException) {
174                ((DependencyResolutionException) t).getResult().getUnresolvedDependencies()
175                        .forEach(d -> failedDependenciesBuilder.add(printDependency(d)));
176            } else {
177                messageBuilder.add(t.getMessage());
178            }
179        }
180
181        String message = Joiner.on("    \n").join(messageBuilder.build());
182        Set<String> failedDependencies = failedDependenciesBuilder.build();
183        if (!failedDependencies.isEmpty()) {
184            if (!message.isEmpty()) {
185                message += "\n";
186            }
187            message += "Could not resolve dependencies: [" + Joiner.on(", ").join(failedDependencies) + "]";
188        }
189        return new MojoExecutionException(message);
190    }
191
192    private static String printDependency(Dependency d) {
193        return d.getArtifact() + " [" + d.getScope() + (d.isOptional() ? ", optional" : "") + "]";
194    }
195
196    /**
197     * Called for any direct project dependency. Factored out from {@link #computeResolutionMap} to allow parallel evaluation of dependencies to speed up the
198     * process.
199     */
200    private void resolveProjectDependency(final Dependency dependency,
201            final ScopeLimitingFilter visibleScopes,
202            final ImmutableSetMultimap.Builder<QualifiedName, VersionResolution> collector)
203            throws MojoExecutionException, DependencyResolutionException, AbstractArtifactResolutionException, VersionRangeResolutionException {
204        final QualifiedName dependencyName = QualifiedName.fromDependency(dependency);
205
206        // see if the resolved, direct dependency contain this name.
207        // If not, the dependency is declared in a scope that is not used (it was filtered out by the scope filter
208        // when the map was created.
209        if (rootDependencyMap.getDirectDependencies().containsKey(dependencyName)) {
210            // a direct dependency
211            final DependencyNode projectDependencyNode = rootDependencyMap.getDirectDependencies().get(dependencyName);
212            assert projectDependencyNode != null;
213
214            checkState(visibleScopes.accept(projectDependencyNode, ImmutableList.of()),
215                    "Dependency %s maps to %s, but scope filter would exclude it. This should never happen!", dependency, projectDependencyNode);
216            computeVersionResolutionForDirectDependency(collector, dependency, projectDependencyNode);
217        }
218
219        // could be a dependency in the full dependency list
220        final DependencyNode projectDependencyNode = rootDependencyMap.getAllDependencies().get(dependencyName);
221        // A project dependency could be e.g. in test scope, but the map has been computed in a narrower scope (e.g. compile)
222        // in that case, it does not contain a dependency node for the dependency. That is ok, simply ignore it.
223        if (projectDependencyNode == null) {
224            return;
225        }
226        checkState(visibleScopes.accept(projectDependencyNode, ImmutableList.of()),
227                "Dependency %s maps to %s, but scope filter would exclude it. This should never happen!", dependency, projectDependencyNode);
228
229        try {
230            // remove the test scope for resolving all the transitive dependencies. Anything that was pulled in in test scope,
231            // now needs its dependencies resolved in compile+runtime scope, not test scope.
232            final ScopeLimitingFilter dependencyScope = ScopeLimitingFilter.computeTransitiveScope(dependency.getScope());
233            computeVersionResolutionForTransitiveDependencies(collector, dependency, projectDependencyNode, dependencyScope);
234        } catch (ProjectBuildingException e) {
235            // This is an optimization and a bug workaround at the same time. Some artifacts exist that
236            // specify a packaging that is not natively supported by maven (e.g. bundle of OSGi bundles), however they
237            // do not bring the necessary extensions to deal with that type. As a result, this causes a "could not read model"
238            // exception. Ignore the transitive dependencies if the project node does not suggest any child artifacts.
239            if (projectDependencyNode.getChildren().isEmpty()) {
240                LOG.debug("Ignoring model building exception for %s, no children were declared", dependency);
241            } else {
242                LOG.warn("Could not read POM for %s, ignoring project and its dependencies!", dependency);
243            }
244        }
245    }
246
247    /**
248     * Create a version resolution for the given direct requestingDependency and artifact.
249     */
250    private void computeVersionResolutionForDirectDependency(
251            final ImmutableSetMultimap.Builder<QualifiedName, VersionResolution> collector,
252            final Dependency requestingDependency,
253            final DependencyNode resolvedDependencyNode)
254            throws AbstractArtifactResolutionException, VersionRangeResolutionException, MojoExecutionException {
255        final QualifiedName requestingDependencyName = QualifiedName.fromDependency(requestingDependency);
256
257        final RepositorySystem repoSystem = context.getRepositorySystem();
258
259        Artifact artifact = convertToPomArtifact(requestingDependency.getArtifact());
260        if (artifact.isSnapshot()) {
261            // convert version of a snapshot artifact to be SNAPSHOT, otherwise the
262            // version range resolver will try to match the timestamp version
263            artifact = artifact.setVersion(artifact.getBaseVersion());
264        }
265
266        final VersionRangeRequest request = context.createVersionRangeRequest(artifact);
267        final VersionRangeResult result = repoSystem.resolveVersionRange(context.getRepositorySystemSession(), request);
268
269        if (!result.getVersions().contains(resolvedDependencyNode.getVersion())) {
270            throw new MojoExecutionException(
271                    format("Cannot determine the recommended version of dependency '%s'; its version specification is '%s', and the resolved version is '%s'.",
272                            requestingDependency, requestingDependency.getArtifact().getBaseVersion(), resolvedDependencyNode.getVersion()));
273        }
274
275        // dependency range contains the project version (or matches it)
276
277        // version from the dependency artifact
278        final ComparableVersion expectedVersion = getVersion(resolvedDependencyNode);
279
280        // this is a direct dependency; it made it through the filter in resolveProjectDependency.
281        final boolean managedDependency = (resolvedDependencyNode.getManagedBits() & DependencyNode.MANAGED_VERSION) != 0;
282        final VersionResolution resolution = VersionResolution.forDirectDependency(QualifiedName.fromProject(context.getRootProject()), expectedVersion,
283                managedDependency);
284
285        if (isIncluded(resolvedDependencyNode, expectedVersion, expectedVersion)) {
286            final Strategy strategy = context.getStrategyCache().forQualifiedName(requestingDependencyName);
287            checkState(strategy != null, "Strategy for %s is null, this should never happen (could not find default strategy?", requestingDependencyName);
288
289            if (!strategy.isCompatible(expectedVersion, expectedVersion)) {
290                resolution.conflict();
291            }
292        } else {
293            LOG.debug("VersionResolution %s is excluded by configuration.", resolution);
294        }
295
296        try {
297            collectorLock.lock();
298            collector.put(requestingDependencyName, resolution);
299        } finally {
300            collectorLock.unlock();
301        }
302    }
303
304    /**
305     * Resolve all transitive dependencies relative to a given dependency, based off the artifact given. A scope filter can be added which limits the results to
306     * the scopes present in that filter.
307     */
308    private void computeVersionResolutionForTransitiveDependencies(
309            final ImmutableSetMultimap.Builder<QualifiedName, VersionResolution> collector,
310            final Dependency requestingDependency,
311            final DependencyNode dependencyNodeForDependency,
312            final DependencyFilter scopeFilter)
313            throws AbstractArtifactResolutionException, ProjectBuildingException, DependencyResolutionException {
314        final AndDependencyFilter filter = new AndDependencyFilter(scopeFilter, new CheckExclusionsFilter(requestingDependency.getExclusions()));
315
316        final DependencyMap dependencyMap = new DependencyMapBuilder(context).mapDependency(dependencyNodeForDependency, filter);
317        final Collection<DependencyNode> transitiveDependencies = dependencyMap.getAllDependencies().values();
318        final QualifiedName requestingDependencyName = QualifiedName.fromDependency(requestingDependency);
319
320        final ImmutableSet<DependencyNode> filteredDependencies = transitiveDependencies.stream()
321                .filter(d -> scopeFilter.accept(d, ImmutableList.of()))
322                .filter(d -> !d.getDependency().isOptional())
323                .collect(toImmutableSet());
324
325        for (final DependencyNode dependencyNode : filteredDependencies) {
326            final QualifiedName dependencyName = QualifiedName.fromDependencyNode(dependencyNode);
327
328            final DependencyNode projectDependencyNode = rootDependencyMap.getAllDependencies().get(dependencyName);
329            if (projectDependencyNode == null) {
330                // the next condition can happen if a dependency is required by one dependency but then overridden by another. e.g.
331                //
332                //   guava (*29.1-jre*, 25.1-android)
333                //    guava 25.1-android depends on org.checkerframework:checker-compat-qual
334                //    guava 29.1-jre depends on org.checkerframework:checker-qual
335                //
336                // as the dependency resolver chose 29.1-jre, only the "checker-qual" dependency will show on the final classpath
337                // however, when resolving all dependencies, the code will resolve the dependency which pulls in guava-25.1-android.
338                // For that dependency, there will be "checker-compat-qual" in the list of dependencies, but when the code tries to
339                // resolve the actual classpath dependency, the "checker-compat-qual" dependency is not in the final classpath.
340                //
341                // This is normal situation and the dependency can just be dropped.
342                //
343                continue;
344            }
345
346            final ComparableVersion resolvedVersion = getVersion(projectDependencyNode);
347            final ComparableVersion expectedVersion = getVersion(dependencyNode);
348
349            final boolean managedDependency = (projectDependencyNode.getManagedBits() & DependencyNode.MANAGED_VERSION) != 0;
350            final VersionResolution resolution = VersionResolution.forTransitiveDependency(requestingDependencyName, expectedVersion, managedDependency);
351
352            if (isIncluded(dependencyNode, expectedVersion, resolvedVersion)) {
353                final Strategy strategy = context.getStrategyCache().forQualifiedName(dependencyName);
354                checkState(strategy != null, "Strategy for %s is null, this should never happen (could not find default strategy?", dependencyName);
355
356                if (!strategy.isCompatible(expectedVersion, resolvedVersion)) {
357                    resolution.conflict();
358                }
359            }
360
361            try {
362                collectorLock.lock();
363                collector.put(dependencyName, resolution);
364            } finally {
365                collectorLock.unlock();
366            }
367        }
368    }
369
370    /**
371     * Returns true if a given artifact and version should be checked.
372     */
373    private boolean isIncluded(DependencyNode dependencyNodeForDependency, ComparableVersion expectedVersion, ComparableVersion resolvedVersion) {
374        return context.getExclusions().stream().noneMatch(exclusion -> exclusion.matches(dependencyNodeForDependency, expectedVersion, resolvedVersion));
375    }
376
377    /**
378     * Return a version object for an Artifact.
379     */
380    private static ComparableVersion getVersion(DependencyNode dependencyNode)
381            throws OverConstrainedVersionException {
382        checkNotNull(dependencyNode, "dependencyNode is null");
383
384        checkState(dependencyNode.getVersion() != null, "DependencyNode %s has a null version selected. Please report a bug!", dependencyNode);
385        return new ComparableVersion(dependencyNode.getVersion().toString());
386    }
387}