View Javadoc
1   /*
2    * Licensed under the Apache License, Version 2.0 (the "License");
3    * you may not use this file except in compliance with the License.
4    * You may obtain a copy of the License at
5    *
6    * http://www.apache.org/licenses/LICENSE-2.0
7    *
8    * Unless required by applicable law or agreed to in writing, software
9    * distributed under the License is distributed on an "AS IS" BASIS,
10   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11   * See the License for the specific language governing permissions and
12   * limitations under the License.
13   */
14  
15  package org.basepom.mojo.duplicatefinder.classpath;
16  
17  import static com.google.common.base.Preconditions.checkNotNull;
18  import static com.google.common.base.Preconditions.checkState;
19  import static java.lang.String.format;
20  
21  import org.basepom.mojo.duplicatefinder.ConflictType;
22  import org.basepom.mojo.duplicatefinder.artifact.MavenCoordinates;
23  
24  import java.io.File;
25  import java.io.IOException;
26  import java.io.InputStream;
27  import java.util.Arrays;
28  import java.util.Collection;
29  import java.util.List;
30  import java.util.Map.Entry;
31  import java.util.Optional;
32  import java.util.concurrent.ConcurrentHashMap;
33  import java.util.concurrent.ConcurrentMap;
34  import java.util.function.Predicate;
35  import java.util.regex.Pattern;
36  import java.util.regex.PatternSyntaxException;
37  import java.util.zip.ZipEntry;
38  import java.util.zip.ZipInputStream;
39  import javax.lang.model.SourceVersion;
40  
41  import com.google.common.annotations.VisibleForTesting;
42  import com.google.common.base.MoreObjects;
43  import com.google.common.base.Splitter;
44  import com.google.common.collect.ImmutableList;
45  import com.google.common.collect.ImmutableList.Builder;
46  import com.google.common.collect.ImmutableMap;
47  import com.google.common.collect.ImmutableMultimap;
48  import com.google.common.collect.Multimap;
49  import com.google.common.collect.MultimapBuilder;
50  import com.google.common.io.Files;
51  import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
52  import org.apache.maven.artifact.Artifact;
53  import org.apache.maven.plugin.MojoExecutionException;
54  import org.apache.maven.project.MavenProject;
55  import org.slf4j.Logger;
56  import org.slf4j.LoggerFactory;
57  
58  @SuppressFBWarnings("NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE")
59  public final class ClasspathDescriptor {
60  
61      private static final Logger LOG = LoggerFactory.getLogger(ClasspathDescriptor.class);
62  
63      private static final MatchPatternPredicate DEFAULT_IGNORED_RESOURCES_PREDICATE = new MatchPatternPredicate(Arrays.asList(
64              // Standard jar folders
65              "^META-INF/.*",
66              "^OSGI-INF/.*",
67              // directory name that shows up all the time
68              "^licenses/.*",
69              // file names that show up all the time
70              ".*license(\\.txt)?$",
71              ".*notice(\\.txt)?$",
72              ".*readme(\\.txt)?$",
73              ".*changelog(\\.txt)?$",
74              ".*third-party(\\.txt)?$",
75              // HTML stuff from javadocs.
76              ".*package\\.html$",
77              ".*overview\\.html$"));
78  
79      @VisibleForTesting
80      static final MatchPatternPredicate DEFAULT_IGNORED_CLASS_PREDICATE = new MatchPatternPredicate(Arrays.asList(
81  
82              "^(.*\\.)?.*\\$.*$",      // matches inner classes in any package
83              "^(.*\\.)?package-info$", // matches package-info in any package
84              "^(.*\\.)?module-info$"   // matches module-info in any package
85      ));
86  
87      private static final MatchPatternPredicate DEFAULT_IGNORED_LOCAL_DIRECTORIES = new MatchPatternPredicate(Arrays.asList(
88              "^.git$",
89              "^.svn$",
90              "^.hg$",
91              "^.bzr$"));
92  
93      /**
94       * This is a global, static cache which can be reused through multiple runs of the plugin in the same VM, e.g. for a multi-module build.
95       */
96      private static final ConcurrentMap<File, ClasspathCacheElement> CACHED_BY_FILE = new ConcurrentHashMap<>();
97  
98      private final Multimap<String, File> classesWithElements = MultimapBuilder.treeKeys().hashSetValues().build();
99      private final Multimap<String, File> resourcesWithElements = MultimapBuilder.treeKeys().hashSetValues().build();
100 
101     private final Predicate<String> resourcesPredicate;
102     private final Predicate<String> classPredicate;
103 
104     private final ImmutableList<Pattern> ignoredResourcePatterns;
105     private final ImmutableList<Pattern> ignoredClassPatterns;
106 
107     public static ClasspathDescriptor createClasspathDescriptor(final MavenProject project,
108             final Multimap<File, Artifact> fileToArtifactMap,
109             final Collection<String> ignoredResourcePatterns,
110             final Collection<String> ignoredClassPatterns,
111             final Collection<MavenCoordinates> ignoredDependencies,
112             final boolean useDefaultResourceIgnoreList,
113             final boolean useDefaultClassIgnoreList,
114             final File... projectFolders) throws MojoExecutionException {
115         checkNotNull(project, "project is null");
116         checkNotNull(fileToArtifactMap, "fileToArtifactMap is null");
117         checkNotNull(ignoredResourcePatterns, "ignoredResourcePatterns is null");
118         checkNotNull(ignoredClassPatterns, "ignoredClassPatterns is null");
119         checkNotNull(ignoredDependencies, "ignoredDependencies is null");
120         checkNotNull(projectFolders, "projectFolders is null");
121 
122         final ClasspathDescriptor classpathDescriptor = new ClasspathDescriptor(useDefaultResourceIgnoreList, ignoredResourcePatterns,
123                 useDefaultClassIgnoreList, ignoredClassPatterns);
124 
125         File file = null;
126 
127         final MatchArtifactPredicate matchArtifactPredicate = new MatchArtifactPredicate(ignoredDependencies);
128 
129         Artifact artifact = null;
130 
131         try {
132             // any entry is either a jar in the repo or a folder in the target folder of a referenced
133             // project. Add the elements that are not ignored by the ignoredDependencies predicate to
134             // the classpath descriptor.
135             for (final Entry<File, Artifact> entry : fileToArtifactMap.entries()) {
136                 artifact = entry.getValue();
137                 file = entry.getKey();
138 
139                 if (file.exists()) {
140                     // Add to the classpath if the artifact predicate does not apply (then it is not in the ignoredDependencies list).
141                     if (!matchArtifactPredicate.apply(artifact)) {
142                         classpathDescriptor.addClasspathElement(file);
143                     }
144                 } else {
145                     // e.g. when running the goal explicitly on a cleaned multi-module project, referenced
146                     // projects will try to use the output folders of a referenced project but these do not
147                     // exist. Obviously, in this case the plugin might return incorrect results (unfortunately
148                     // false negatives, but there is not much it can do here (besides fail the build here with a
149                     // cryptic error message. Maybe add a flag?).
150                     LOG.debug(format("Classpath element '%s' does not exist.", file.getAbsolutePath()));
151                 }
152             }
153         } catch (final IOException ex) {
154             throw new MojoExecutionException(format("Error trying to access file '%s' for artifact '%s'", file, artifact), ex);
155         }
156 
157         try {
158             // Add project folders unconditionally.
159             for (final File projectFile : projectFolders) {
160                 file = projectFile;
161                 if (projectFile.exists()) {
162                     classpathDescriptor.addClasspathElement(file);
163                 } else {
164                     // See above. This may happen in the project has been cleaned before running the goal directly.
165                     LOG.debug(format("Project folder '%s' does not exist.", file.getAbsolutePath()));
166                 }
167             }
168         } catch (final IOException ex) {
169             throw new MojoExecutionException(format("Error trying to access project folder '%s'", file), ex);
170         }
171 
172         return classpathDescriptor;
173     }
174 
175     private ClasspathDescriptor(final boolean useDefaultResourceIgnoreList,
176             final Collection<String> ignoredResourcePatterns,
177             final boolean useDefaultClassIgnoreList,
178             final Collection<String> ignoredClassPatterns)
179             throws MojoExecutionException {
180         final Builder<Pattern> ignoredResourcePatternsBuilder = ImmutableList.builder();
181 
182         // build resource predicate...
183         Predicate<String> resourcesPredicate = s -> false;
184 
185         // predicate matching the default ignores
186         if (useDefaultResourceIgnoreList) {
187             resourcesPredicate = resourcesPredicate.or(DEFAULT_IGNORED_RESOURCES_PREDICATE);
188             ignoredResourcePatternsBuilder.addAll(DEFAULT_IGNORED_RESOURCES_PREDICATE.getPatterns());
189         }
190 
191         if (!ignoredResourcePatterns.isEmpty()) {
192             try {
193                 // predicate matching the user ignores
194                 MatchPatternPredicate ignoredResourcesPredicate = new MatchPatternPredicate(ignoredResourcePatterns);
195                 resourcesPredicate = resourcesPredicate.or(ignoredResourcesPredicate);
196                 ignoredResourcePatternsBuilder.addAll(ignoredResourcesPredicate.getPatterns());
197             } catch (final PatternSyntaxException pse) {
198                 throw new MojoExecutionException("Error compiling resourceIgnore pattern: " + pse.getMessage());
199             }
200         }
201 
202         this.resourcesPredicate = resourcesPredicate;
203         this.ignoredResourcePatterns = ignoredResourcePatternsBuilder.build();
204 
205         final Builder<Pattern> ignoredClassPatternsBuilder = ImmutableList.builder();
206 
207         // build class predicate.
208         Predicate<String> classPredicate = s -> false;
209 
210         // predicate matching the default ignores
211         if (useDefaultClassIgnoreList) {
212             classPredicate = classPredicate.or(DEFAULT_IGNORED_CLASS_PREDICATE);
213             ignoredClassPatternsBuilder.addAll(DEFAULT_IGNORED_CLASS_PREDICATE.getPatterns());
214         }
215 
216         if (!ignoredClassPatterns.isEmpty()) {
217             try {
218                 // predicate matching the user ignores
219                 MatchPatternPredicate ignoredPackagePredicate = new MatchPatternPredicate(ignoredClassPatterns);
220                 classPredicate = classPredicate.or(ignoredPackagePredicate);
221                 ignoredClassPatternsBuilder.addAll(ignoredPackagePredicate.getPatterns());
222             } catch (final PatternSyntaxException pse) {
223                 throw new MojoExecutionException("Error compiling classIgnore pattern: " + pse.getMessage());
224             }
225         }
226 
227         this.classPredicate = classPredicate;
228         this.ignoredClassPatterns = ignoredClassPatternsBuilder.build();
229     }
230 
231     public ImmutableMap<String, Collection<File>> getClasspathElementLocations(final ConflictType type) {
232         checkNotNull(type, "type is null");
233         switch (type) {
234             case CLASS:
235                 return ImmutableMultimap.copyOf(classesWithElements).asMap();
236             case RESOURCE:
237                 return ImmutableMultimap.copyOf(resourcesWithElements).asMap();
238             default:
239                 throw new IllegalStateException("Type '" + type + "' unknown!");
240         }
241     }
242 
243     public ImmutableList<Pattern> getIgnoredResourcePatterns() {
244         return ignoredResourcePatterns;
245     }
246 
247     public ImmutableList<Pattern> getIgnoredClassPatterns() {
248         return ignoredClassPatterns;
249     }
250 
251     public ImmutableList<Pattern> getIgnoredDirectoryPatterns() {
252         return DEFAULT_IGNORED_LOCAL_DIRECTORIES.getPatterns();
253     }
254 
255 
256     private void addClasspathElement(final File element) throws IOException {
257         checkState(element.exists(), "Path '%s' does not exist!", element.getAbsolutePath());
258 
259         ClasspathCacheElement cached = CACHED_BY_FILE.get(element);
260 
261         if (cached == null) {
262             final ClasspathCacheElement.Builder cacheBuilder = ClasspathCacheElement.builder(element);
263             if (element.isDirectory()) {
264                 addDirectory(cacheBuilder, element, new PackageNameHolder());
265             } else {
266                 addArchive(cacheBuilder, element);
267             }
268             final ClasspathCacheElement newCached = cacheBuilder.build();
269             final ClasspathCacheElement oldCached = CACHED_BY_FILE.putIfAbsent(element, newCached);
270             cached = MoreObjects.firstNonNull(oldCached, newCached);
271         } else {
272             LOG.debug(format("Cache hit for '%s'", element.getAbsolutePath()));
273         }
274 
275         cached.putResources(resourcesWithElements, resourcesPredicate);
276         cached.putClasses(classesWithElements, classPredicate);
277 
278     }
279 
280     private void addDirectory(final ClasspathCacheElement.Builder cacheBuilder, final File workDir, final PackageNameHolder packageName) {
281         final File[] files = workDir.listFiles();
282 
283         if (files != null) {
284             for (final File file : files) {
285                 if (file.isDirectory()) {
286                     if (DEFAULT_IGNORED_LOCAL_DIRECTORIES.apply(file.getName())) {
287                         LOG.debug(format("Ignoring local directory '%s'", file.getAbsolutePath()));
288                     } else {
289                         addDirectory(cacheBuilder, file, packageName.getChildPackage(file.getName()));
290                     }
291 
292                 } else if (file.isFile()) {
293                     if ("class".equals(Files.getFileExtension(file.getName()))) {
294                         final String className = packageName.getQualifiedName(Files.getNameWithoutExtension(file.getName()));
295                         cacheBuilder.addClass(className);
296                     } else {
297                         final String resourcePath = packageName.getQualifiedPath(file.getName());
298                         cacheBuilder.addResource(resourcePath);
299                     }
300                 } else {
301                     LOG.warn(format("Ignoring unknown file type for '%s'", file.getAbsolutePath()));
302                 }
303             }
304         }
305     }
306 
307     private void addArchive(final ClasspathCacheElement.Builder cacheBuilder, final File element) throws IOException {
308 
309         try (
310                 InputStream input = element.toURI().toURL().openStream();
311                 ZipInputStream zipInput = new ZipInputStream(input)) {
312 
313             ZipEntry entry;
314 
315             while ((entry = zipInput.getNextEntry()) != null) {
316                 if (!entry.isDirectory()) {
317                     final String name = entry.getName();
318                     Optional<List<String>> validatedElements = validateClassName(name);
319                     if (validatedElements.isPresent()) {
320                         List<String> nameElements = validatedElements.get();
321                         final PackageNameHolder packageName = new PackageNameHolder(nameElements.subList(0, nameElements.size() - 1));
322                         final String className = packageName.getQualifiedName(Files.getNameWithoutExtension(name));
323                         cacheBuilder.addClass(className);
324                     } else {
325                         final String resourcePath = name.replace('\\', File.separatorChar);
326                         cacheBuilder.addResource(resourcePath);
327                     }
328                 }
329             }
330         }
331     }
332 
333     @VisibleForTesting
334     static Optional<List<String>> validateClassName(String fullClassPath) {
335         if (fullClassPath == null) {
336             return Optional.empty();
337         }
338 
339         // ZIP/Jars always use "/" as separators
340         final List<String> nameElements = ImmutableList.copyOf(Splitter.on("/").splitToList(fullClassPath));
341         if (nameElements.isEmpty()) {
342             LOG.warn(format("ZIP entry '%s' split into empty list!", fullClassPath));
343             return Optional.empty();
344         }
345         String classFileName = nameElements.get(nameElements.size() - 1);
346 
347         if (!"class".equals(Files.getFileExtension(classFileName))) {
348             LOG.debug(format("Ignoring %s, %s is not a class file", fullClassPath, classFileName));
349             return Optional.empty();
350         }
351         for (int i = 0; i < nameElements.size() - 1; i++) {
352             if (!SourceVersion.isIdentifier(nameElements.get(i))) {
353                 LOG.debug(format("Ignoring %s, %s is not a valid package element", fullClassPath, nameElements.get(i)));
354                 return Optional.empty();
355             }
356         }
357 
358         String className = Files.getNameWithoutExtension(classFileName);
359         if (!(SourceVersion.isIdentifier(className)
360                 || "module-info".equals(className)
361                 || "package-info".equals(className))) {
362             LOG.debug(format("Ignoring %s, %s is not a valid class identifier", fullClassPath, className));
363             return Optional.empty();
364         }
365 
366         return Optional.of(nameElements);
367     }
368 }