Merge branch 'gerrit/mad-hatter'
Change-Id: I300a5e2910bde9e7bcd5f187efca26e4c7689ffb
diff --git a/asterixdb/src/main/licenses/templates/asterix-license.ftl b/asterixdb/src/main/licenses/templates/asterix-license.ftl
index 381d400..434ce44 100644
--- a/asterixdb/src/main/licenses/templates/asterix-license.ftl
+++ b/asterixdb/src/main/licenses/templates/asterix-license.ftl
@@ -44,6 +44,9 @@
<#assign isare = "are"/>
</#if>
<#list projects as p>
+ <#if p.shadowed>
+ <#continue/>
+ </#if>
* ${p.name} (${p.groupId}:${p.artifactId}:${p.version})
<#list p.locations as loc>
- ${loc}${p.jarName}
diff --git a/asterixdb/src/main/licenses/templates/asterix-notice.ftl b/asterixdb/src/main/licenses/templates/asterix-notice.ftl
index c825397..5090cb0 100644
--- a/asterixdb/src/main/licenses/templates/asterix-notice.ftl
+++ b/asterixdb/src/main/licenses/templates/asterix-notice.ftl
@@ -34,6 +34,9 @@
<#assign projects = e.getValue()/>
<#list projects as p>
+ <#if p.shadowed>
+ <#continue/>
+ </#if>
${p.name} (${p.groupId}:${p.artifactId}:${p.version})
<#list p.locations as loc>
- ${loc}${p.jarName}
diff --git a/hyracks-fullstack/hyracks-fullstack-license/src/main/licenses/templates/hyracks-notice.ftl b/hyracks-fullstack/hyracks-fullstack-license/src/main/licenses/templates/hyracks-notice.ftl
index 8a2eb05..0cf9378 100644
--- a/hyracks-fullstack/hyracks-fullstack-license/src/main/licenses/templates/hyracks-notice.ftl
+++ b/hyracks-fullstack/hyracks-fullstack-license/src/main/licenses/templates/hyracks-notice.ftl
@@ -30,6 +30,9 @@
<#assign noticeText = e.getKey()/>
<#assign projects = e.getValue()/>
<#list projects as p>
+ <#if p.shadowed>
+ <#continue/>
+ </#if>
${p.name} (${p.groupId}:${p.artifactId}:${p.version})
<#list p.locations as loc>
- ${loc}${p.jarName}
diff --git a/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/DownloadLicensesMojo.java b/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/DownloadLicensesMojo.java
index 293c08c..c8980cd 100644
--- a/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/DownloadLicensesMojo.java
+++ b/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/DownloadLicensesMojo.java
@@ -63,7 +63,7 @@
String fileName = entry.getLicense().getContentFile(false);
doDownload(timeoutMillis, i, url, fileName);
});
- } catch (ProjectBuildingException e) {
+ } catch (ProjectBuildingException | IOException e) {
throw new MojoExecutionException("Unexpected exception: " + e, e);
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/GenerateFileMojo.java b/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/GenerateFileMojo.java
index 6bc92b8..6d30a69 100644
--- a/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/GenerateFileMojo.java
+++ b/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/GenerateFileMojo.java
@@ -39,6 +39,7 @@
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.Properties;
import java.util.Set;
import java.util.SortedMap;
import java.util.SortedSet;
@@ -52,15 +53,22 @@
import java.util.regex.Pattern;
import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.tuple.Pair;
import org.apache.hyracks.maven.license.freemarker.IndentDirective;
import org.apache.hyracks.maven.license.freemarker.LoadFileDirective;
import org.apache.hyracks.maven.license.project.LicensedProjects;
import org.apache.hyracks.maven.license.project.Project;
+import org.apache.maven.artifact.Artifact;
+import org.apache.maven.artifact.DefaultArtifact;
+import org.apache.maven.artifact.handler.ArtifactHandler;
+import org.apache.maven.artifact.handler.DefaultArtifactHandler;
import org.apache.maven.plugin.MojoExecutionException;
import org.apache.maven.plugin.MojoFailureException;
import org.apache.maven.plugins.annotations.Mojo;
import org.apache.maven.plugins.annotations.Parameter;
import org.apache.maven.plugins.annotations.ResolutionScope;
+import org.apache.maven.project.MavenProject;
import org.apache.maven.project.ProjectBuildingException;
import com.fasterxml.jackson.databind.ObjectMapper;
@@ -106,6 +114,9 @@
@Parameter
private boolean stripFoundationAssertionFromNotices = false;
+ @Parameter
+ private boolean validateShadowLicenses = false;
+
private SortedMap<String, SortedSet<Project>> noticeMap;
@java.lang.Override
@@ -139,11 +150,12 @@
}
licenseSpecs.addAll(urlToLicenseMap.values());
for (LicenseSpec license : licenseSpecs) {
- resolveArtifactContent(license, true);
+ resolveArtifactContent(license, true, false);
}
}
- private String resolveArtifactContent(ArtifactSpec artifact, boolean bestEffort) throws IOException {
+ private String resolveArtifactContent(ArtifactSpec artifact, boolean bestEffort, boolean suppressWarning)
+ throws IOException {
if (artifact.getContent() == null) {
getLog().debug("Resolving content for " + artifact.getUrl() + " (" + artifact.getContentFile() + ")");
File cFile = new File(artifact.getContentFile());
@@ -157,7 +169,9 @@
}
if (!cFile.exists()) {
if (!bestEffort) {
- getLog().warn("MISSING: content file (" + cFile + ") for url: " + artifact.getUrl());
+ if (!suppressWarning) {
+ getLog().warn("MISSING: content file (" + cFile + ") for url: " + artifact.getUrl());
+ }
artifact.setContent("MISSING: " + artifact.getContentFile() + " (" + artifact.getUrl() + ")");
}
} else {
@@ -264,10 +278,12 @@
for (Project p : lps.getProjects()) {
String licenseText = p.getLicenseText();
if (licenseText == null) {
- warnUnlessFlag(p.gav(), IGNORE_MISSING_EMBEDDED_LICENSE,
- "Using license other than from within artifact: " + p.gav() + " (" + lps.getLicense()
- + ")");
- licenseText = resolveArtifactContent(lps.getLicense(), false);
+ if (validateProjectLicense(p)) {
+ warnUnlessFlag(p.gav(), IGNORE_MISSING_EMBEDDED_LICENSE,
+ "Using license other than from within artifact: " + p.gav() + " (" + lps.getLicense()
+ + ")");
+ }
+ licenseText = resolveArtifactContent(lps.getLicense(), false, !validateProjectLicense(p));
}
LicenseSpec spec = lps.getLicense();
if (spec.getDisplayName() == null) {
@@ -293,6 +309,10 @@
licenseMap = licenseMap2;
}
+ private boolean validateProjectLicense(Project p) {
+ return !p.isShadowed() || validateShadowLicenses;
+ }
+
private Set<Project> getProjects() {
Set<Project> projects = new HashSet<>();
licenseMap.values().forEach(p -> projects.addAll(p.getProjects()));
@@ -305,9 +325,11 @@
String noticeText = p.getNoticeText();
if (noticeText == null && noticeOverrides.containsKey(p.gav())) {
String noticeUrl = noticeOverrides.get(p.gav());
- warnUnlessFlag(p.gav(), IGNORE_NOTICE_OVERRIDE,
- "Using notice other than from within artifact: " + p.gav() + " (" + noticeUrl + ")");
- p.setNoticeText(resolveArtifactContent(new NoticeSpec(noticeUrl), false));
+ if (validateProjectLicense(p)) {
+ warnUnlessFlag(p.gav(), IGNORE_NOTICE_OVERRIDE,
+ "Using notice other than from within artifact: " + p.gav() + " (" + noticeUrl + ")");
+ }
+ p.setNoticeText(resolveArtifactContent(new NoticeSpec(noticeUrl), false, p.isShadowed()));
} else if (noticeText == null && !noticeOverrides.containsKey(p.gav())
&& Boolean.TRUE.equals(getProjectFlag(p.gav(), IGNORE_NOTICE_OVERRIDE))) {
getLog().warn(p + " has IGNORE_NOTICE_OVERRIDE flag set, but no override defined...");
@@ -339,7 +361,7 @@
// this seems way too liberal
resolveArtifactFiles("NOTICE", IGNORE_MISSING_EMBEDDED_NOTICE, ALTERNATE_NOTICE_FILE,
entry -> entry.getName().matches("(.*/|^)" + "NOTICE" + "(.(txt|md))?"), Project::setNoticeText,
- text -> stripFoundationAssertionFromNotices ? FOUNDATION_PATTERN.matcher(text).replaceAll("") : text);
+ getNoticeFileContentTransformer(), !validateShadowLicenses);
}
private void resolveLicenseFiles() throws MojoExecutionException, IOException {
@@ -347,13 +369,13 @@
// this seems way too liberal
resolveArtifactFiles("LICENSE", IGNORE_MISSING_EMBEDDED_LICENSE, ALTERNATE_LICENSE_FILE,
entry -> entry.getName().matches("(.*/|^)" + "LICENSE" + "(.(txt|md))?"), Project::setLicenseText,
- UnaryOperator.identity());
+ UnaryOperator.identity(), !validateShadowLicenses);
}
private void resolveArtifactFiles(final String name, final ProjectFlag ignoreFlag,
final ProjectFlag alternateFilenameFlag, final Predicate<JarEntry> filter,
- final BiConsumer<Project, String> consumer, final UnaryOperator<String> contentTransformer)
- throws MojoExecutionException, IOException {
+ final BiConsumer<Project, String> consumer, final UnaryOperator<String> contentTransformer,
+ boolean skipShadowed) throws MojoExecutionException, IOException {
for (Project p : getProjects()) {
File artifactFile = new File(p.getArtifactPath());
if (!artifactFile.exists()) {
@@ -361,6 +383,9 @@
} else if (!artifactFile.getName().endsWith(".jar")) {
getLog().info("Skipping unknown artifact file type: " + artifactFile);
continue;
+ } else if (skipShadowed && p.isShadowed()) {
+ getLog().info("Skipping shadowed project: " + p.gav());
+ continue;
}
String alternateFilename = (String) getProjectFlag(p.gav(), alternateFilenameFlag);
Predicate<JarEntry> finalFilter =
@@ -406,4 +431,71 @@
}
return matches;
}
+
+ private UnaryOperator<String> getNoticeFileContentTransformer() {
+ UnaryOperator<String> transformer;
+ if (stripFoundationAssertionFromNotices) {
+ transformer = text -> FOUNDATION_PATTERN.matcher(text).replaceAll("");
+ } else {
+ transformer = UnaryOperator.identity();
+ }
+ return transformer;
+ }
+
+ @java.lang.Override
+ protected void gatherProjectDependencies(MavenProject project,
+ Map<MavenProject, List<Pair<String, String>>> dependencyLicenseMap,
+ Map<String, MavenProject> dependencyGavMap) throws ProjectBuildingException, MojoExecutionException {
+ super.gatherProjectDependencies(project, dependencyLicenseMap, dependencyGavMap);
+ gatherShadowedDependencies(dependencyLicenseMap, dependencyGavMap);
+ }
+
+ @java.lang.Override
+ protected void processExtraDependencies(Map<MavenProject, List<Pair<String, String>>> dependencyLicenseMap,
+ Map<String, MavenProject> dependencyGavMap) throws ProjectBuildingException, MojoExecutionException {
+ super.processExtraDependencies(dependencyLicenseMap, dependencyGavMap);
+ gatherShadowedDependencies(dependencyLicenseMap, dependencyGavMap);
+ }
+
+ private void gatherShadowedDependencies(Map<MavenProject, List<Pair<String, String>>> dependencyLicenseMap,
+ Map<String, MavenProject> dependencyGavMap) throws MojoExecutionException, ProjectBuildingException {
+ Set<MavenProject> projects = new TreeSet<>(Comparator.comparing(MavenProject::getId));
+ projects.addAll(dependencyLicenseMap.keySet());
+ for (MavenProject p : projects) {
+ boolean finished = false;
+ File artifactFile = p.getArtifact().getFile();
+ if (!artifactFile.exists()) {
+ throw new MojoExecutionException("Artifact file " + artifactFile + " does not exist!");
+ } else if (!artifactFile.getName().endsWith(".jar")) {
+ getLog().info("Skipping unknown artifact file type: " + artifactFile);
+ finished = true;
+ }
+ if (!finished) {
+ try (JarFile jarFile = new JarFile(artifactFile)) {
+ SortedMap<String, JarEntry> matches = gatherMatchingEntries(jarFile,
+ entry -> entry.getName().matches("(.*/|^)" + "pom\\.properties"));
+ if (!matches.isEmpty()) {
+ for (JarEntry entry : matches.values()) {
+ Properties props = new Properties();
+ props.load(jarFile.getInputStream(entry));
+ String groupId = props.getProperty("groupId");
+ String artifactId = props.getProperty("artifactId");
+ String version = props.getProperty("version");
+ String gav = groupId + ":" + artifactId + ":" + version;
+ if (!dependencyGavMap.containsKey(gav)) {
+ getLog().info("adding " + gav + " (shadowed from " + p.getId() + ")");
+ ArtifactHandler handler = new DefaultArtifactHandler("jar");
+ String[] gavParts = StringUtils.split(gav, ':');
+ Artifact manualDep = new DefaultArtifact(gavParts[0], gavParts[1], gavParts[2],
+ Artifact.SCOPE_COMPILE, "jar", null, handler);
+ processArtifact(manualDep, dependencyLicenseMap, dependencyGavMap, true);
+ }
+ }
+ }
+ } catch (IOException e) {
+ throw new MojoExecutionException(e);
+ }
+ }
+ }
+ }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/LicenseMojo.java b/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/LicenseMojo.java
index 2206621..73cc695 100644
--- a/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/LicenseMojo.java
+++ b/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/LicenseMojo.java
@@ -70,6 +70,7 @@
public abstract class LicenseMojo extends AbstractMojo {
+ private static final String SHADOWED_KEY = LicenseMojo.class.getName() + "_shadowed";
@Parameter
protected List<Override> overrides = new ArrayList<>();
@@ -249,7 +250,7 @@
});
}
- protected void addDependenciesToLicenseMap() throws ProjectBuildingException {
+ protected void addDependenciesToLicenseMap() throws ProjectBuildingException, MojoExecutionException, IOException {
Map<MavenProject, List<Pair<String, String>>> dependencyLicenseMap = gatherDependencies();
dependencyLicenseMap.forEach((depProject, value) -> {
Set<String> locations = dependencySets.isEmpty() ? Collections.singleton(location)
@@ -310,7 +311,9 @@
licenseUrl = fakeLicenseUrl;
}
}
- addProject(new Project(depProject, depLocation, depProject.getArtifact().getFile()),
+ addProject(
+ new Project(depProject, depLocation, depProject.getArtifact().getFile(),
+ Boolean.parseBoolean(String.valueOf(depProject.getContextValue(SHADOWED_KEY)))),
new LicenseSpec(licenseUrl, displayName), true);
}
@@ -363,11 +366,13 @@
}
}
- protected Map<MavenProject, List<Pair<String, String>>> gatherDependencies() throws ProjectBuildingException {
+ protected Map<MavenProject, List<Pair<String, String>>> gatherDependencies()
+ throws ProjectBuildingException, MojoExecutionException, IOException {
Map<MavenProject, List<Pair<String, String>>> dependencyLicenseMap = new HashMap<>();
Map<String, MavenProject> dependencyGavMap = new HashMap<>();
gatherProjectDependencies(project, dependencyLicenseMap, dependencyGavMap);
+ processExtraDependencies(dependencyLicenseMap, dependencyGavMap);
for (Override override : overrides) {
// Collect both <gav></gav> and <gavs><gav></gav><gav></gav>...</gavs>
@@ -399,27 +404,40 @@
return dependencyLicenseMap;
}
- private void gatherProjectDependencies(MavenProject project,
+ protected void gatherProjectDependencies(MavenProject project,
Map<MavenProject, List<Pair<String, String>>> dependencyLicenseMap,
- Map<String, MavenProject> dependencyGavMap) throws ProjectBuildingException {
+ Map<String, MavenProject> dependencyGavMap) throws ProjectBuildingException, MojoExecutionException {
final Set<Artifact> dependencyArtifacts = project.getArtifacts();
if (dependencyArtifacts != null) {
for (Artifact depArtifact : dependencyArtifacts) {
- processArtifact(depArtifact, dependencyLicenseMap, dependencyGavMap);
+ processArtifact(depArtifact, dependencyLicenseMap, dependencyGavMap, false);
}
}
- for (String gav : extraDependencies) {
- ArtifactHandler handler = new DefaultArtifactHandler("jar");
- String[] gavParts = StringUtils.split(gav, ':');
- Artifact manualDep = new DefaultArtifact(gavParts[0], gavParts[1], gavParts[2], Artifact.SCOPE_COMPILE,
- "jar", null, handler);
- processArtifact(manualDep, dependencyLicenseMap, dependencyGavMap);
- }
}
- private void processArtifact(Artifact depArtifact,
+ protected void processExtraDependencies(Map<MavenProject, List<Pair<String, String>>> dependencyLicenseMap,
+ Map<String, MavenProject> dependencyGavMap) throws ProjectBuildingException, MojoExecutionException {
+ for (String extraDependency : extraDependencies) {
+ String[] gavParts = StringUtils.split(extraDependency, ':');
+ String gav = gavParts[0] + ":" + gavParts[1] + ":" + gavParts[2];
+ if (!dependencyGavMap.containsKey(gav)) {
+ ArtifactHandler handler = new DefaultArtifactHandler("jar");
+ Artifact manualDep = new DefaultArtifact(gavParts[0], gavParts[1], gavParts[2], Artifact.SCOPE_COMPILE,
+ "jar", null, handler);
+ boolean shadowed = false;
+ if (gavParts.length > 3) {
+ shadowed = "shadowed".equals(gavParts[3]);
+ }
+ processArtifact(manualDep, dependencyLicenseMap, dependencyGavMap, shadowed);
+ } else {
+ getLog().warn("not adding extra dependency " + gav + " as it is already added");
+ }
+ }
+ }
+
+ protected void processArtifact(Artifact depArtifact,
Map<MavenProject, List<Pair<String, String>>> dependencyLicenseMap,
- Map<String, MavenProject> dependencyGavMap) throws ProjectBuildingException {
+ Map<String, MavenProject> dependencyGavMap, boolean shadowed) throws ProjectBuildingException {
if (!excludedScopes.contains(depArtifact.getScope())) {
MavenProject dep = resolveDependency(depArtifact);
if (!depArtifact.isResolved()) {
@@ -442,6 +460,7 @@
: (license1.getName() != null ? license1.getName() : "LICENSE_EMPTY_NAME_URL");
licenseUrls.add(new ImmutablePair<>(url, license1.getName()));
}
+ dep.setContextValue(SHADOWED_KEY, shadowed);
dependencyLicenseMap.put(dep, licenseUrls);
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/project/Project.java b/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/project/Project.java
index e44914d..a755c05 100644
--- a/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/project/Project.java
+++ b/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/project/Project.java
@@ -41,6 +41,7 @@
private String licenseText;
private String sourcePointer;
private String classifier;
+ private final boolean shadowed;
@JsonIgnore
private MavenProject mavenProject;
@@ -48,7 +49,7 @@
public static final Comparator<Project> PROJECT_COMPARATOR =
(o1, o2) -> o1.compareToken().compareTo(o2.compareToken());
- public Project(MavenProject project, String location, File artifactPath) {
+ public Project(MavenProject project, String location, File artifactPath, boolean shadowed) {
mavenProject = project;
name = project.getName();
groupId = project.getGroupId();
@@ -57,6 +58,7 @@
url = project.getUrl();
classifier = project.getArtifact().getClassifier();
this.artifactPath = artifactPath.getPath();
+ this.shadowed = shadowed;
setLocation(location);
}
@@ -65,7 +67,8 @@
@JsonProperty("artifactId") String artifactId, @JsonProperty("url") String url,
@JsonProperty("version") String version, @JsonProperty("location") String location,
@JsonProperty("artifactPath") String artifactPath, @JsonProperty("noticeText") String noticeText,
- @JsonProperty("licenseText") String licenseText, @JsonProperty("classifier") String classifier) {
+ @JsonProperty("licenseText") String licenseText, @JsonProperty("classifier") String classifier,
+ @JsonProperty("shadowed") boolean shadowed) {
this.name = name;
this.groupId = groupId;
this.artifactId = artifactId;
@@ -76,6 +79,7 @@
this.noticeText = noticeText;
this.licenseText = licenseText;
this.classifier = classifier;
+ this.shadowed = shadowed;
}
public String getName() {
@@ -185,6 +189,10 @@
this.sourcePointer = sourcePointer;
}
+ public boolean isShadowed() {
+ return shadowed;
+ }
+
@Override
public String toString() {
return "Project [" + gav() + "]";