ASTERIXDB-1320, ASTERIXDB-1323: License Fixes

ASTERIXDB-1320:
- LICENSE lists a large number of CDDL licensed bits of software
  CDDL is Category B we should provide a link to the source code (see 3.1.
  Availability of Source Code in (1)). Previous advice on legal-discuss
  was this goes in NOTICE but recent discussions have left this a bit more muddled.

ASTERIXDB-1323:
- Missing normalize.css (MIT) ./asterix-examples/src/main/resources/admaql101-demo
  /static/css/bootstrap.min.css
- Missing license for second bottle file (MIT) ./asterix-examples/src/main/
  resources/tweetbook-demo/bottle.py
- Bootstrap version bundled is Apache licensed not MIT licensed ./asterix-app/src/
  main/resources/webui/static/js/bootstrap.min.js
- It’s also not mentioned for all licenses what each license is (MIT/BSD etc) that
  can be helpful. The version of the bundled software is also helpful.
- Should include text of RainbowVis-JS license (or better still a pointer to a copy
  of the license file) [5] not a pointer to a URL on github

Also, misc cleanup.

Change-Id: Ie9fe9c18f63624896ccda420e1bf83ae0127021e
Reviewed-on: https://asterix-gerrit.ics.uci.edu/1463
Sonar-Qube: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
BAD: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Ian Maxon <imaxon@apache.org>
diff --git a/hyracks-fullstack/NOTICE b/hyracks-fullstack/NOTICE
index 4888468..d44f7a0 100644
--- a/hyracks-fullstack/NOTICE
+++ b/hyracks-fullstack/NOTICE
@@ -1,4 +1,4 @@
-Apache AsterixDB Hyracks and Algebricks
+Apache Hyracks and Algebricks
 Copyright 2015-2017 The Apache Software Foundation
 
 This product includes software developed at
diff --git a/hyracks-fullstack/hyracks-fullstack-license/pom.xml b/hyracks-fullstack/hyracks-fullstack-license/pom.xml
index 201a628..7326284 100644
--- a/hyracks-fullstack/hyracks-fullstack-license/pom.xml
+++ b/hyracks-fullstack/hyracks-fullstack-license/pom.xml
@@ -59,8 +59,7 @@
               <outputFile>LICENSE</outputFile>
             </generatedFile>
             <generatedFile>
-              <!-- TODO(mblow): share the template with asterixdb as a maven artifact -->
-              <template>asterix-notice.ftl</template>
+              <template>hyracks-notice.ftl</template>
               <outputFile>NOTICE</outputFile>
             </generatedFile>
           </generatedFiles>
@@ -83,7 +82,7 @@
           <templateProperties>
             <hyracksControlCcLocation />
             <hyracksControlCcResourcesPrefix>hyracks/hyracks-control/hyracks-control-cc/src/main/resources/</hyracksControlCcResourcesPrefix>
-            <packageName>Hyracks and Algebricks</packageName>
+            <packageName>Apache Hyracks and Algebricks</packageName>
           </templateProperties>
         </configuration>
       </plugin>
diff --git a/hyracks-fullstack/hyracks-fullstack-license/src/main/licenses/templates/asterix-notice.ftl b/hyracks-fullstack/hyracks-fullstack-license/src/main/licenses/templates/hyracks-notice.ftl
similarity index 87%
rename from hyracks-fullstack/hyracks-fullstack-license/src/main/licenses/templates/asterix-notice.ftl
rename to hyracks-fullstack/hyracks-fullstack-license/src/main/licenses/templates/hyracks-notice.ftl
index 6b0570c..c59e80a 100644
--- a/hyracks-fullstack/hyracks-fullstack-license/src/main/licenses/templates/asterix-notice.ftl
+++ b/hyracks-fullstack/hyracks-fullstack-license/src/main/licenses/templates/hyracks-notice.ftl
@@ -17,23 +17,20 @@
  ! under the License.
 -->
 <#-- TODO(mblow): share notice file template with asterixdb via maven artifact -->
-<#if packageName?has_content>
-Apache AsterixDB ${packageName!}
-<#else>
-Apache AsterixDB
-</#if>
+Apache Hyracks and Algebricks
 Copyright 2015-2017 The Apache Software Foundation
 
 This product includes software developed at
 The Apache Software Foundation (http://www.apache.org/).
 <#list noticeMap>
 
-AsterixDB utilizes many libraries, which come with the following applicable NOTICE(s):
+Hyracks and Algebricks utilize many libraries, which come with the following applicable NOTICE(s):
 
 <#items as e>
    <#assign noticeText = e.getKey()/>
    <#assign projects = e.getValue()/>
    <#list projects as p>
+${p.name} (${p.groupId}:${p.artifactId}:${p.version})
        <#list p.locations as loc>
 - ${loc}${p.artifactId}-${p.version}.jar
        </#list>
diff --git a/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/pom.xml b/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/pom.xml
index 36c4edf..193f9e8 100644
--- a/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/pom.xml
@@ -85,6 +85,11 @@
       <artifactId>maven-artifact</artifactId>
       <version>3.0</version>
     </dependency>
+    <dependency>
+      <groupId>org.apache.maven</groupId>
+      <artifactId>maven-compat</artifactId>
+      <version>3.3.9</version>
+    </dependency>
   </dependencies>
 
 </project>
\ No newline at end of file
diff --git a/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/GenerateFileMojo.java b/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/GenerateFileMojo.java
index fa5429a..b604f18 100644
--- a/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/GenerateFileMojo.java
+++ b/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/GenerateFileMojo.java
@@ -22,8 +22,11 @@
 import java.io.FileWriter;
 import java.io.IOException;
 import java.io.StringWriter;
+import java.lang.Override;
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
 import java.util.Comparator;
 import java.util.Enumeration;
 import java.util.HashMap;
@@ -31,6 +34,7 @@
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.Random;
 import java.util.Set;
 import java.util.SortedMap;
 import java.util.SortedSet;
@@ -49,17 +53,31 @@
 import freemarker.template.Configuration;
 import freemarker.template.Template;
 import freemarker.template.TemplateException;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOUtils;
 import org.apache.hyracks.maven.license.freemarker.IndentDirective;
 import org.apache.hyracks.maven.license.freemarker.LoadFileDirective;
 import org.apache.hyracks.maven.license.project.LicensedProjects;
 import org.apache.hyracks.maven.license.project.Project;
-import org.apache.commons.io.IOUtils;
+import org.apache.maven.artifact.Artifact;
+import org.apache.maven.artifact.DefaultArtifact;
+import org.apache.maven.artifact.metadata.ArtifactMetadata;
+import org.apache.maven.artifact.repository.ArtifactRepository;
+import org.apache.maven.artifact.repository.ArtifactRepositoryPolicy;
+import org.apache.maven.artifact.repository.Authentication;
+import org.apache.maven.artifact.repository.DefaultRepositoryRequest;
+import org.apache.maven.artifact.repository.layout.ArtifactRepositoryLayout;
+import org.apache.maven.artifact.repository.layout.DefaultRepositoryLayout;
+import org.apache.maven.artifact.resolver.ArtifactResolutionRequest;
+import org.apache.maven.artifact.resolver.ArtifactResolutionResult;
 import org.apache.maven.plugin.MojoExecutionException;
 import org.apache.maven.plugin.MojoFailureException;
 import org.apache.maven.plugins.annotations.Mojo;
 import org.apache.maven.plugins.annotations.Parameter;
 import org.apache.maven.plugins.annotations.ResolutionScope;
+import org.apache.maven.project.MavenProject;
 import org.apache.maven.project.ProjectBuildingException;
+import org.apache.maven.repository.Proxy;
 
 @Mojo(name = "generate",
         requiresProject = true,
@@ -109,15 +127,68 @@
             resolveNoticeFiles();
             resolveLicenseFiles();
             rebuildLicenseContentProjectMap();
-            buildNoticeProjectMap();
-            persistLicenseMap();
             combineCommonGavs();
+            collectSourceAssemblies();
+            persistLicenseMap();
+            buildNoticeProjectMap();
             generateFiles();
         } catch (IOException | TemplateException | ProjectBuildingException e) {
             throw new MojoExecutionException("Unexpected exception: " + e, e);
         }
     }
 
+    private void collectSourceAssemblies() throws ProjectBuildingException, IOException {
+        try (StubArtifactRepository stubRepo = new StubArtifactRepository()) {
+            DefaultRepositoryRequest rr = new DefaultRepositoryRequest();
+            rr.setLocalRepository(stubRepo);
+            ArtifactRepository central = getCentralRepository();
+            rr.setRemoteRepositories(Collections.singletonList(central));
+            ArtifactResolutionRequest request = new ArtifactResolutionRequest(rr);
+            for (LicensedProjects lp : licenseMap.values()) {
+                if (lp.getLicense().getDisplayName() != null
+                        && lp.getLicense().getDisplayName().toLowerCase().contains("cddl")) {
+                    ensureCDDLSourcesPointer(lp.getProjects(), central, request);
+                }
+            }
+        }
+    }
+
+    private void ensureCDDLSourcesPointer(Collection<Project> projects, ArtifactRepository central,
+                                          ArtifactResolutionRequest request) throws ProjectBuildingException {
+        for (Project p : projects) {
+            if (p.getSourcePointer() != null) {
+                continue;
+            }
+            getLog().debug("finding sources for artifact: " + p);
+            Artifact sourcesArtifact = new DefaultArtifact(p.getGroupId(), p.getArtifactId(),
+                    p.getVersion(), Artifact.SCOPE_COMPILE, "jar", "sources", null);
+            MavenProject mavenProject = resolveDependency(sourcesArtifact);
+            sourcesArtifact.setArtifactHandler(mavenProject.getArtifact().getArtifactHandler());
+
+            request.setArtifact(sourcesArtifact);
+            ArtifactResolutionResult result = artifactResolver.resolve(request);
+            getLog().debug("result: " + result);
+            StringBuilder noticeBuilder = new StringBuilder("You may obtain ");
+            noticeBuilder.append(p.getName()).append(" in Source Code form code here:\n");
+            if (result.isSuccess()) {
+                noticeBuilder.append(central.getUrl()).append("/").append(central.pathOf(sourcesArtifact));
+            } else {
+                getLog().warn("Unable to find sources in 'central' for " + p + ", falling back to project url: "
+                        + p.getUrl());
+                noticeBuilder.append(p.getUrl() != null ? p.getUrl() : "MISSING SOURCE POINTER");
+            }
+            p.setSourcePointer(noticeBuilder.toString());
+        }
+    }
+
+    private ArtifactRepository getCentralRepository() {
+        for (ArtifactRepository repo : session.getRequest().getRemoteRepositories()) {
+            if ("central".equals(repo.getId())) {
+                return repo;
+            }
+        }
+        throw new IllegalStateException("Unable to find 'central' remote repository!");
+    }
 
     private void resolveLicenseContent() throws IOException {
         Set<LicenseSpec> licenseSpecs = new HashSet<>();
@@ -220,9 +291,9 @@
                         spec.setDisplayName(projects.getLicense().getDisplayName());
                     }
                 }
-                for (Project project : projects.getProjects()) {
-                    project.setLocation(extraLicenseFile.getLocation());
-                    addProject(project, projects.getLicense(), extraLicenseFile.isAdditive());
+                for (Project p : projects.getProjects()) {
+                    p.setLocation(extraLicenseFile.getLocation());
+                    addProject(p, projects.getLicense(), extraLicenseFile.isAdditive());
                 }
             }
         }
@@ -244,10 +315,10 @@
         int counter = 0;
         Map<String, LicensedProjects> licenseMap2 = new TreeMap<>(WHITESPACE_NORMALIZED_COMPARATOR);
         for (LicensedProjects lps : licenseMap.values()) {
-            for (Project project : lps.getProjects()) {
-                String licenseText = project.getLicenseText();
+            for (Project p : lps.getProjects()) {
+                String licenseText = p.getLicenseText();
                 if (licenseText == null) {
-                    getLog().warn("Using license other than from within artifact: " + project.gav());
+                    getLog().warn("Using license other than from within artifact: " + p.gav());
                     licenseText = resolveLicenseContent(lps.getLicense(), false);
                 }
                 LicenseSpec spec = lps.getLicense();
@@ -268,7 +339,7 @@
                 if (lp2.getLicense().getDisplayName() == null) {
                     lp2.getLicense().setDisplayName(lps.getLicense().getDisplayName());
                 }
-                lp2.addProject(project);
+                lp2.addProject(p);
             }
         }
         licenseMap = licenseMap2;
@@ -282,15 +353,26 @@
 
     private void buildNoticeProjectMap() {
         noticeMap = new TreeMap<>(WHITESPACE_NORMALIZED_COMPARATOR);
-        for (Project project : getProjects()) {
-            final String noticeText = project.getNoticeText();
+        for (Project p : getProjects()) {
+            prependSourcePointerToNotice(p);
+            final String noticeText = p.getNoticeText();
             if (noticeText == null) {
                 continue;
             }
             if (!noticeMap.containsKey(noticeText)) {
                 noticeMap.put(noticeText, new TreeSet<>(Project.PROJECT_COMPARATOR));
             }
-            noticeMap.get(noticeText).add(project);
+            noticeMap.get(noticeText).add(p);
+        }
+    }
+
+    private void prependSourcePointerToNotice(Project project) {
+        if (project.getSourcePointer() != null) {
+            String notice = project.getSourcePointer().replace("\n", "\n    ");
+            if (project.getNoticeText() != null) {
+                notice += "\n\n" + project.getNoticeText();
+            }
+            project.setNoticeText(notice);
         }
     }
 
@@ -308,8 +390,8 @@
     private void resolveArtifactFiles(final String name, Predicate<JarEntry> filter,
                                       BiConsumer<Project, String> consumer, UnaryOperator<String> contentTransformer)
             throws MojoExecutionException, IOException {
-        for (Project project : getProjects()) {
-            File artifactFile = new File(project.getArtifactPath());
+        for (Project p : getProjects()) {
+            File artifactFile = new File(p.getArtifactPath());
             if (!artifactFile.exists()) {
                 throw new MojoExecutionException("Artifact file " + artifactFile + " does not exist!");
             } else if (!artifactFile.getName().endsWith(".jar")) {
@@ -320,15 +402,15 @@
                 SortedMap<String, JarEntry> matches = gatherMatchingEntries(jarFile,
                         filter);
                 if (matches.isEmpty()) {
-                    getLog().warn("No " + name + " file found for " + project.gav());
+                    getLog().warn("No " + name + " file found for " + p.gav());
                 } else {
                     if (matches.size() > 1) {
-                        getLog().warn("Multiple " + name + " files found for " + project.gav() + ": " + matches.keySet()
+                        getLog().warn("Multiple " + name + " files found for " + p.gav() + ": " + matches.keySet()
                                 + "; taking first");
                     } else {
-                        getLog().info(project.gav() + " has " + name + " file: " + matches.keySet());
+                        getLog().info(p.gav() + " has " + name + " file: " + matches.keySet());
                     }
-                    resolveContent(project, jarFile, matches.values().iterator().next(),
+                    resolveContent(p, jarFile, matches.values().iterator().next(),
                             contentTransformer, consumer, name);
                 }
             }
@@ -359,5 +441,154 @@
         }
         return matches;
     }
+
+    private static class StubArtifactRepository implements ArtifactRepository, AutoCloseable {
+        private static final Random random = new Random();
+        private final File tempDir;
+        private final ArtifactRepositoryLayout layout;
+
+        public StubArtifactRepository() {
+            String tmpDir = System.getProperty("java.io.tmpdir", "/tmp");
+            this.tempDir = new File(tmpDir, "repo" + random.nextInt());
+            this.layout = new DefaultRepositoryLayout();
+        }
+
+        @Override
+        public ArtifactRepositoryLayout getLayout() {
+            return layout;
+        }
+
+        @Override
+        public String pathOf(Artifact artifact) {
+            return this.layout.pathOf(artifact);
+        }
+
+        @Override
+        public String getBasedir() {
+            return tempDir.toString();
+        }
+
+        @Override
+        public void close() throws IOException {
+            FileUtils.deleteDirectory(tempDir);
+
+        }
+
+        @Override
+        public String pathOfRemoteRepositoryMetadata(ArtifactMetadata artifactMetadata) {
+            return null;
+        }
+
+        @Override
+        public String pathOfLocalRepositoryMetadata(ArtifactMetadata artifactMetadata,
+                                                    ArtifactRepository artifactRepository) {
+            return null;
+        }
+
+        @Override
+        public String getUrl() {
+            return null;
+        }
+
+        @Override
+        public void setUrl(String s) {
+            // unused
+        }
+
+        @Override
+        public String getProtocol() {
+            return null;
+        }
+
+        @Override
+        public String getId() {
+            return "stub";
+        }
+
+        @Override
+        public void setId(String s) {
+            // unused
+        }
+
+        @Override
+        public ArtifactRepositoryPolicy getSnapshots() {
+            return null;
+        }
+
+        @Override
+        public void setSnapshotUpdatePolicy(ArtifactRepositoryPolicy artifactRepositoryPolicy) {
+            // unused
+        }
+
+        @Override
+        public ArtifactRepositoryPolicy getReleases() {
+            return null;
+        }
+
+        @Override
+        public void setReleaseUpdatePolicy(ArtifactRepositoryPolicy artifactRepositoryPolicy) {
+            // unused
+        }
+
+        @Override
+        public void setLayout(ArtifactRepositoryLayout artifactRepositoryLayout) {
+            // unused
+        }
+
+        @Override
+        public String getKey() {
+            return null;
+        }
+
+        @Override
+        public boolean isUniqueVersion() {
+            return false;
+        }
+
+        @Override
+        public boolean isBlacklisted() {
+            return false;
+        }
+
+        @Override
+        public void setBlacklisted(boolean b) {
+            // unused
+        }
+
+        @Override
+        public Artifact find(Artifact artifact) {
+            return null;
+        }
+
+        @Override
+        public List<String> findVersions(Artifact artifact) {
+            return Collections.emptyList();
+        }
+
+        @Override
+        public boolean isProjectAware() {
+            return false;
+        }
+
+        @Override
+        public void setAuthentication(Authentication authentication) {
+            // unused
+        }
+
+        @Override
+        public Authentication getAuthentication() {
+            return null;
+        }
+
+        @Override
+        public void setProxy(Proxy proxy) {
+            // unused
+        }
+
+        @Override
+        public Proxy getProxy() {
+            return null;
+        }
+    }
 }
 
diff --git a/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/LicenseMojo.java b/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/LicenseMojo.java
index 4363170..929b7d7 100644
--- a/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/LicenseMojo.java
+++ b/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/LicenseMojo.java
@@ -40,6 +40,7 @@
 import org.apache.hyracks.maven.license.project.Project;
 import org.apache.maven.artifact.Artifact;
 import org.apache.maven.artifact.repository.ArtifactRepository;
+import org.apache.maven.execution.MavenSession;
 import org.apache.maven.model.License;
 import org.apache.maven.model.Model;
 import org.apache.maven.plugin.AbstractMojo;
@@ -86,6 +87,12 @@
     @Component
     private ModelInheritanceAssembler assembler;
 
+    @Parameter( defaultValue = "${session}", required = true, readonly = true )
+    protected MavenSession session;
+
+    @Component
+    protected org.apache.maven.artifact.resolver.ArtifactResolver artifactResolver;
+
     @Parameter ( required = true )
     private String location;
 
@@ -192,6 +199,9 @@
                     urlToLicenseMap.put(alias ,license);
                 }
             }
+        } else if (license.getDisplayName() == null && spec.getDisplayName() != null) {
+            getLog().info("Propagating license name from " + project.gav() + ": " + spec.getDisplayName());
+            license.setDisplayName(spec.getDisplayName());
         }
         licenseUrl = license.getUrl();
         LicensedProjects entry = licenseMap.get(licenseUrl);
diff --git a/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/project/Project.java b/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/project/Project.java
index e5e57ad..80d4548 100644
--- a/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/project/Project.java
+++ b/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/project/Project.java
@@ -19,7 +19,9 @@
 package org.apache.hyracks.maven.license.project;
 
 import java.io.File;
+import java.util.Arrays;
 import java.util.Comparator;
+import java.util.List;
 
 import org.apache.maven.project.MavenProject;
 
@@ -37,6 +39,7 @@
     private String artifactPath;
     private String noticeText;
     private String licenseText;
+    private String sourcePointer;
 
     @JsonIgnore
     private MavenProject mavenProject;
@@ -105,8 +108,9 @@
     }
 
     @JsonIgnore
-    public String [] getLocations() {
-        return getLocation().split(",");
+    public List<String> getLocations() {
+        // TODO(mblow): store locations as an set instead of string
+        return Arrays.asList(getLocation().split(","));
     }
 
     public void setArtifactId(String artifactId) {
@@ -156,4 +160,17 @@
     public void setLicenseText(String licenseText) {
         this.licenseText = licenseText;
     }
+
+    public String getSourcePointer() {
+        return sourcePointer;
+    }
+
+    public void setSourcePointer(String sourcePointer) {
+        this.sourcePointer = sourcePointer;
+    }
+
+    @Override
+    public String toString() {
+        return "Project [" + gav() + "]";
+    }
 }