Merge "Merge branch 'neo'"
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/LoadRecordFieldsRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/LoadRecordFieldsRule.java
index b9d512b..42cce52 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/LoadRecordFieldsRule.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/LoadRecordFieldsRule.java
@@ -27,6 +27,7 @@
import org.apache.asterix.algebra.base.OperatorAnnotation;
import org.apache.asterix.common.exceptions.CompilationException;
import org.apache.asterix.common.exceptions.ErrorCode;
+import org.apache.asterix.lang.common.util.FunctionUtil;
import org.apache.asterix.om.base.AInt32;
import org.apache.asterix.om.base.AString;
import org.apache.asterix.om.constants.AsterixConstantValue;
@@ -64,7 +65,7 @@
public class LoadRecordFieldsRule implements IAlgebraicRewriteRule {
- private ExtractFieldLoadExpressionVisitor exprVisitor = new ExtractFieldLoadExpressionVisitor();
+ private final ExtractFieldLoadExpressionVisitor exprVisitor = new ExtractFieldLoadExpressionVisitor();
@Override
public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
@@ -98,13 +99,13 @@
// checking if we can annotate a Selection as using just one field
// access
SelectOperator sigma = (SelectOperator) op1;
- LinkedList<LogicalVariable> vars = new LinkedList<LogicalVariable>();
+ List<LogicalVariable> vars = new ArrayList<>();
VariableUtilities.getUsedVariables(sigma, vars);
if (vars.size() == 1) {
// we can annotate Selection
AssignOperator assign1 = (AssignOperator) op1.getInputs().get(0).getValue();
- AbstractLogicalExpression expr1 = (AbstractLogicalExpression) getFirstExpr(assign1);
- if (expr1.getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) {
+ ILogicalExpression expr1 = getFirstExpr(assign1);
+ if (FunctionUtil.isFieldAccessFunction(expr1)) {
AbstractFunctionCallExpression f = (AbstractFunctionCallExpression) expr1;
// f should be a call to a field/data access kind of
// function
@@ -141,7 +142,7 @@
}
// create an assign
LogicalVariable v = context.newVar();
- AssignOperator a2 = new AssignOperator(v, new MutableObject<ILogicalExpression>(f));
+ AssignOperator a2 = new AssignOperator(v, new MutableObject<>(f));
a2.setSourceLocation(expr.getSourceLocation());
pushFieldAssign(a2, topOp, context);
context.computeAndSetTypeEnvironmentForOperator(a2);
@@ -151,7 +152,7 @@
LogicalVariable var = ref.getVariableReference();
List<LogicalVariable> keys = context.findPrimaryKey(var);
if (keys != null) {
- List<LogicalVariable> tail = new ArrayList<LogicalVariable>();
+ List<LogicalVariable> tail = new ArrayList<>();
tail.add(v);
FunctionalDependency pk = new FunctionalDependency(keys, tail);
context.addPrimaryKey(pk);
@@ -408,5 +409,4 @@
private static ILogicalExpression getFirstExpr(AssignOperator assign) {
return assign.getExpressions().get(0).getValue();
}
-
}
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/ObjectsQueries.xml b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/ObjectsQueries.xml
index 2954227..77fc7c5 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/ObjectsQueries.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/ObjectsQueries.xml
@@ -248,4 +248,9 @@
<expected-warn>Duplicate field name 'fname1' (in line 25, at column 45)</expected-warn>
</compilation-unit>
</test-case>
+ <test-case FilePath="objects">
+ <compilation-unit name="load-record-fields">
+ <output-dir compare="Text">load-record-fields</output-dir>
+ </compilation-unit>
+ </test-case>
</test-group>
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/load-record-fields/load-record-fields.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/load-record-fields/load-record-fields.1.ddl.sqlpp
new file mode 100644
index 0000000..99898f9
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/load-record-fields/load-record-fields.1.ddl.sqlpp
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+DROP DATAVERSE test IF EXISTS;
+CREATE DATAVERSE test;
+
+USE test;
+
+CREATE TYPE OpenType AS {
+ id: int
+};
+
+CREATE DATASET MyDataset(OpenType)
+PRIMARY KEY id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/load-record-fields/load-record-fields.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/load-record-fields/load-record-fields.2.update.sqlpp
new file mode 100644
index 0000000..1f80ae1
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/load-record-fields/load-record-fields.2.update.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+INSERT INTO MyDataset (
+ {"id": 1, "name": "Alice"},
+ {"id": 2, "name": "Bob"}
+);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/load-record-fields/load-record-fields.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/load-record-fields/load-record-fields.3.query.sqlpp
new file mode 100644
index 0000000..278f5d2
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/load-record-fields/load-record-fields.3.query.sqlpp
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+-- Disabled for a simpler plan
+SET `compiler.sort.parallel` "false";
+
+
+SELECT VALUE md.name
+FROM MyDataset md
+LET currentData = {"myDate": current_date()}
+WHERE currentData.myDate = current_date()
+ORDER BY md.id
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/load-record-fields/load-record-fields.4.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/load-record-fields/load-record-fields.4.query.sqlpp
new file mode 100644
index 0000000..f44921f
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/objects/load-record-fields/load-record-fields.4.query.sqlpp
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+-- Disabled for a simpler plan
+SET `compiler.sort.parallel` "false";
+
+EXPLAIN
+SELECT VALUE md.name
+FROM MyDataset md
+LET currentData = {"myDate": current_date()}
+WHERE currentData.myDate = current_date()
+ORDER BY md.id
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/objects/load-record-fields/load-record-fields.3.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/objects/load-record-fields/load-record-fields.3.adm
new file mode 100644
index 0000000..ac2dc97
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/objects/load-record-fields/load-record-fields.3.adm
@@ -0,0 +1,2 @@
+"Alice"
+"Bob"
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/objects/load-record-fields/load-record-fields.4.plan b/asterixdb/asterix-app/src/test/resources/runtimets/results/objects/load-record-fields/load-record-fields.4.plan
new file mode 100644
index 0000000..5ac69f3
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/objects/load-record-fields/load-record-fields.4.plan
@@ -0,0 +1,26 @@
+distribute result [$$28]
+-- DISTRIBUTE_RESULT |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ project ([$$28])
+ -- STREAM_PROJECT |PARTITIONED|
+ exchange
+ -- SORT_MERGE_EXCHANGE [$$30(ASC) ] |PARTITIONED|
+ project ([$$28, $$30])
+ -- STREAM_PROJECT |PARTITIONED|
+ select (eq($$31, current-date()))
+ -- STREAM_SELECT |PARTITIONED|
+ assign [$$31] <- [current-date()]
+ -- ASSIGN |PARTITIONED|
+ project ([$$30, $$28])
+ -- STREAM_PROJECT |PARTITIONED|
+ assign [$$28] <- [$$md.getField("name")]
+ -- ASSIGN |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ data-scan []<-[$$30, $$md] <- test.MyDataset
+ -- DATASOURCE_SCAN |PARTITIONED|
+ exchange
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ empty-tuple-source
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/FunctionUtil.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/FunctionUtil.java
index 42a35f3..1cf88c2 100644
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/FunctionUtil.java
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/FunctionUtil.java
@@ -58,6 +58,7 @@
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.algebricks.common.utils.Triple;
import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import org.apache.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
import org.apache.hyracks.api.exceptions.IWarningCollector;
@@ -303,4 +304,16 @@
function.getSignature(), e.getMessage());
}
}
+
+ public static boolean isFieldAccessFunction(ILogicalExpression expression) {
+ if (expression.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
+ return false;
+ }
+
+ AbstractFunctionCallExpression funcExpr = (AbstractFunctionCallExpression) expression;
+ FunctionIdentifier fid = funcExpr.getFunctionIdentifier();
+
+ return BuiltinFunctions.FIELD_ACCESS_BY_INDEX.equals(fid) || BuiltinFunctions.FIELD_ACCESS_BY_NAME.equals(fid)
+ || BuiltinFunctions.FIELD_ACCESS_NESTED.equals(fid);
+ }
}
diff --git a/asterixdb/src/main/licenses/templates/asterix-license.ftl b/asterixdb/src/main/licenses/templates/asterix-license.ftl
index 381d400..f9c969e 100644
--- a/asterixdb/src/main/licenses/templates/asterix-license.ftl
+++ b/asterixdb/src/main/licenses/templates/asterix-license.ftl
@@ -34,8 +34,11 @@
<#list licenseMap as e>
<#assign licenseUrl = e.getKey()/>
<#assign entry = e.getValue()/>
- <#assign projects = entry.projects/>
+ <#assign projects = entry.nonShadowedProjects/>
<#assign license = entry.getLicense()/>
+ <#if projects?size == 0>
+ <#continue/>
+ </#if>
<#if projects?size == 1>
Component:
<#assign isare = "is"/>
diff --git a/asterixdb/src/main/licenses/templates/asterix-notice.ftl b/asterixdb/src/main/licenses/templates/asterix-notice.ftl
index c825397..5090cb0 100644
--- a/asterixdb/src/main/licenses/templates/asterix-notice.ftl
+++ b/asterixdb/src/main/licenses/templates/asterix-notice.ftl
@@ -34,6 +34,9 @@
<#assign projects = e.getValue()/>
<#list projects as p>
+ <#if p.shadowed>
+ <#continue/>
+ </#if>
${p.name} (${p.groupId}:${p.artifactId}:${p.version})
<#list p.locations as loc>
- ${loc}${p.jarName}
diff --git a/hyracks-fullstack/hyracks-fullstack-license/src/main/licenses/templates/hyracks-notice.ftl b/hyracks-fullstack/hyracks-fullstack-license/src/main/licenses/templates/hyracks-notice.ftl
index 8a2eb05..0cf9378 100644
--- a/hyracks-fullstack/hyracks-fullstack-license/src/main/licenses/templates/hyracks-notice.ftl
+++ b/hyracks-fullstack/hyracks-fullstack-license/src/main/licenses/templates/hyracks-notice.ftl
@@ -30,6 +30,9 @@
<#assign noticeText = e.getKey()/>
<#assign projects = e.getValue()/>
<#list projects as p>
+ <#if p.shadowed>
+ <#continue/>
+ </#if>
${p.name} (${p.groupId}:${p.artifactId}:${p.version})
<#list p.locations as loc>
- ${loc}${p.jarName}
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/NodeHeartbeatWork.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/NodeHeartbeatWork.java
index b772ef9..6fa6224 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/NodeHeartbeatWork.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/NodeHeartbeatWork.java
@@ -28,9 +28,12 @@
import org.apache.hyracks.control.common.heartbeat.HeartbeatData;
import org.apache.hyracks.control.common.ipc.NodeControllerRemoteProxy;
import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
public class NodeHeartbeatWork extends AbstractHeartbeatWork {
+ private static final Logger LOGGER = LogManager.getLogger();
private final InetSocketAddress ncAddress;
public NodeHeartbeatWork(ClusterControllerService ccs, String nodeId, HeartbeatData hbData,
@@ -47,6 +50,8 @@
ncState.getNodeController().heartbeatAck(ccs.getCcId(), null);
} else {
// unregistered nc- let him know
+ LOGGER.info("received a heartbeat from unregistered node {}; sending negative ack to node address {}",
+ nodeId, ncAddress);
NodeControllerRemoteProxy nc =
new NodeControllerRemoteProxy(ccs.getCcId(), ccs.getClusterIPC().getReconnectingHandle(ncAddress));
nc.heartbeatAck(ccs.getCcId(), HyracksDataException.create(ErrorCode.NO_SUCH_NODE, nodeId));
diff --git a/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/DownloadLicensesMojo.java b/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/DownloadLicensesMojo.java
index 293c08c..c8980cd 100644
--- a/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/DownloadLicensesMojo.java
+++ b/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/DownloadLicensesMojo.java
@@ -63,7 +63,7 @@
String fileName = entry.getLicense().getContentFile(false);
doDownload(timeoutMillis, i, url, fileName);
});
- } catch (ProjectBuildingException e) {
+ } catch (ProjectBuildingException | IOException e) {
throw new MojoExecutionException("Unexpected exception: " + e, e);
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/GenerateFileMojo.java b/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/GenerateFileMojo.java
index 347c19a..2c7a23b 100644
--- a/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/GenerateFileMojo.java
+++ b/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/GenerateFileMojo.java
@@ -43,6 +43,7 @@
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.Properties;
import java.util.Set;
import java.util.SortedMap;
import java.util.SortedSet;
@@ -56,15 +57,22 @@
import java.util.regex.Pattern;
import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.tuple.Pair;
import org.apache.hyracks.maven.license.freemarker.IndentDirective;
import org.apache.hyracks.maven.license.freemarker.LoadFileDirective;
import org.apache.hyracks.maven.license.project.LicensedProjects;
import org.apache.hyracks.maven.license.project.Project;
+import org.apache.maven.artifact.Artifact;
+import org.apache.maven.artifact.DefaultArtifact;
+import org.apache.maven.artifact.handler.ArtifactHandler;
+import org.apache.maven.artifact.handler.DefaultArtifactHandler;
import org.apache.maven.plugin.MojoExecutionException;
import org.apache.maven.plugin.MojoFailureException;
import org.apache.maven.plugins.annotations.Mojo;
import org.apache.maven.plugins.annotations.Parameter;
import org.apache.maven.plugins.annotations.ResolutionScope;
+import org.apache.maven.project.MavenProject;
import org.apache.maven.project.ProjectBuildingException;
import com.fasterxml.jackson.databind.ObjectMapper;
@@ -110,6 +118,11 @@
@Parameter
private boolean stripFoundationAssertionFromNotices = false;
+ @Parameter
+ private boolean includeShadowedDependencies = true;
+ @Parameter
+ private boolean validateShadowLicenses = false;
+
private SortedMap<String, SortedSet<Project>> noticeMap;
@java.lang.Override
@@ -143,11 +156,12 @@
}
licenseSpecs.addAll(urlToLicenseMap.values());
for (LicenseSpec license : licenseSpecs) {
- resolveArtifactContent(license, true);
+ resolveArtifactContent(license, true, false);
}
}
- private String resolveArtifactContent(ArtifactSpec artifact, boolean bestEffort) throws IOException {
+ private String resolveArtifactContent(ArtifactSpec artifact, boolean bestEffort, boolean suppressWarning)
+ throws IOException {
if (artifact.getContent() == null) {
getLog().debug("Resolving content for " + artifact.getUrl() + " (" + artifact.getContentFile() + ")");
File cFile = new File(artifact.getContentFile());
@@ -161,7 +175,9 @@
}
if (!cFile.exists()) {
if (!bestEffort) {
- getLog().warn("MISSING: content file (" + cFile + ") for url: " + artifact.getUrl());
+ if (!suppressWarning) {
+ getLog().warn("MISSING: content file (" + cFile + ") for url: " + artifact.getUrl());
+ }
artifact.setContent("MISSING: " + artifact.getContentFile() + " (" + artifact.getUrl() + ")");
}
} else {
@@ -268,10 +284,12 @@
for (Project p : lps.getProjects()) {
String licenseText = p.getLicenseText();
if (licenseText == null) {
- warnUnlessFlag(p.gav(), IGNORE_MISSING_EMBEDDED_LICENSE,
- "Using license other than from within artifact: " + p.gav() + " (" + lps.getLicense()
- + ")");
- licenseText = resolveArtifactContent(lps.getLicense(), false);
+ if (validateProjectLicense(p)) {
+ warnUnlessFlag(p.gav(), IGNORE_MISSING_EMBEDDED_LICENSE,
+ "Using license other than from within artifact: " + p.gav() + " (" + lps.getLicense()
+ + ")");
+ }
+ licenseText = resolveArtifactContent(lps.getLicense(), false, !validateProjectLicense(p));
}
LicenseSpec spec = lps.getLicense();
if (spec.getDisplayName() == null) {
@@ -297,6 +315,10 @@
licenseMap = licenseMap2;
}
+ private boolean validateProjectLicense(Project p) {
+ return !p.isShadowed() || validateShadowLicenses;
+ }
+
private Set<Project> getProjects() {
Set<Project> projects = new HashSet<>();
licenseMap.values().forEach(p -> projects.addAll(p.getProjects()));
@@ -309,9 +331,11 @@
String noticeText = p.getNoticeText();
if (noticeText == null && noticeOverrides.containsKey(p.gav())) {
String noticeUrl = noticeOverrides.get(p.gav());
- warnUnlessFlag(p.gav(), IGNORE_NOTICE_OVERRIDE,
- "Using notice other than from within artifact: " + p.gav() + " (" + noticeUrl + ")");
- p.setNoticeText(resolveArtifactContent(new NoticeSpec(noticeUrl), false));
+ if (validateProjectLicense(p)) {
+ warnUnlessFlag(p.gav(), IGNORE_NOTICE_OVERRIDE,
+ "Using notice other than from within artifact: " + p.gav() + " (" + noticeUrl + ")");
+ }
+ p.setNoticeText(resolveArtifactContent(new NoticeSpec(noticeUrl), false, p.isShadowed()));
} else if (noticeText == null && !noticeOverrides.containsKey(p.gav())
&& Boolean.TRUE.equals(getProjectFlag(p.gav(), IGNORE_NOTICE_OVERRIDE))) {
getLog().warn(p + " has IGNORE_NOTICE_OVERRIDE flag set, but no override defined...");
@@ -370,7 +394,7 @@
// this seems way too liberal
filter = entry -> entry.getName().matches("(.*/|^)" + "NOTICE" + "(.(txt|md))?");
consumer = Project::setNoticeText;
- contentTransformer = UnaryOperator.identity();
+ contentTransformer = getNoticeFileContentTransformer();
break;
case LICENSE:
name = "LICENSE";
@@ -381,8 +405,7 @@
// this seems way too liberal
filter = entry -> entry.getName().matches("(.*/|^)" + "LICENSE" + "(.(txt|md))?");
consumer = Project::setLicenseText;
- contentTransformer = stripFoundationAssertionFromNotices
- ? text -> FOUNDATION_PATTERN.matcher(text).replaceAll("") : UnaryOperator.identity();
+ contentTransformer = UnaryOperator.identity();
break;
default:
throw new IllegalStateException("NYI: " + artifact);
@@ -394,6 +417,9 @@
} else if (!artifactFile.getName().endsWith(".jar")) {
getLog().info("Skipping unknown artifact file type: " + artifactFile);
continue;
+ } else if (!validateShadowLicenses && p.isShadowed()) {
+ getLog().info("Skipping shadowed project: " + p.gav());
+ continue;
}
String alternateFilename = (String) getProjectFlag(p.gav(), alternateFilenameFlag);
Predicate<JarEntry> finalFilter =
@@ -472,4 +498,76 @@
}
return matches;
}
+
+ private UnaryOperator<String> getNoticeFileContentTransformer() {
+ UnaryOperator<String> transformer;
+ if (stripFoundationAssertionFromNotices) {
+ transformer = text -> FOUNDATION_PATTERN.matcher(text).replaceAll("");
+ } else {
+ transformer = UnaryOperator.identity();
+ }
+ return transformer;
+ }
+
+ @java.lang.Override
+ protected void gatherProjectDependencies(MavenProject project,
+ Map<MavenProject, List<Pair<String, String>>> dependencyLicenseMap,
+ Map<String, MavenProject> dependencyGavMap) throws ProjectBuildingException, MojoExecutionException {
+ super.gatherProjectDependencies(project, dependencyLicenseMap, dependencyGavMap);
+ gatherShadowedDependencies(dependencyLicenseMap, dependencyGavMap);
+ }
+
+ @java.lang.Override
+ protected void processExtraDependencies(Map<MavenProject, List<Pair<String, String>>> dependencyLicenseMap,
+ Map<String, MavenProject> dependencyGavMap) throws ProjectBuildingException, MojoExecutionException {
+ super.processExtraDependencies(dependencyLicenseMap, dependencyGavMap);
+ gatherShadowedDependencies(dependencyLicenseMap, dependencyGavMap);
+ }
+
+ private void gatherShadowedDependencies(Map<MavenProject, List<Pair<String, String>>> dependencyLicenseMap,
+ Map<String, MavenProject> dependencyGavMap) throws MojoExecutionException, ProjectBuildingException {
+ Set<MavenProject> projects = new TreeSet<>(Comparator.comparing(MavenProject::getId));
+ if (!includeShadowedDependencies) {
+ getLog().info("Not gathering shadowed dependencies as 'includeShadowedDependencies' is set to "
+ + includeShadowedDependencies);
+ return;
+ }
+ projects.addAll(dependencyLicenseMap.keySet());
+ for (MavenProject p : projects) {
+ boolean finished = false;
+ File artifactFile = p.getArtifact().getFile();
+ if (!artifactFile.exists()) {
+ throw new MojoExecutionException("Artifact file " + artifactFile + " does not exist!");
+ } else if (!artifactFile.getName().endsWith(".jar")) {
+ getLog().info("Skipping unknown artifact file type: " + artifactFile);
+ finished = true;
+ }
+ if (!finished) {
+ try (JarFile jarFile = new JarFile(artifactFile)) {
+ SortedMap<String, JarEntry> matches = gatherMatchingEntries(jarFile,
+ entry -> entry.getName().matches("(.*/|^)" + "pom\\.properties"));
+ if (!matches.isEmpty()) {
+ for (JarEntry entry : matches.values()) {
+ Properties props = new Properties();
+ props.load(jarFile.getInputStream(entry));
+ String groupId = props.getProperty("groupId");
+ String artifactId = props.getProperty("artifactId");
+ String version = props.getProperty("version");
+ String gav = groupId + ":" + artifactId + ":" + version;
+ if (!dependencyGavMap.containsKey(gav)) {
+ getLog().info("adding " + gav + " (shadowed from " + p.getId() + ")");
+ ArtifactHandler handler = new DefaultArtifactHandler("jar");
+ String[] gavParts = StringUtils.split(gav, ':');
+ Artifact manualDep = new DefaultArtifact(gavParts[0], gavParts[1], gavParts[2],
+ Artifact.SCOPE_COMPILE, "jar", null, handler);
+ processArtifact(manualDep, dependencyLicenseMap, dependencyGavMap, true);
+ }
+ }
+ }
+ } catch (IOException e) {
+ throw new MojoExecutionException(e);
+ }
+ }
+ }
+ }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/LicenseMojo.java b/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/LicenseMojo.java
index 2206621..73cc695 100644
--- a/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/LicenseMojo.java
+++ b/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/LicenseMojo.java
@@ -70,6 +70,7 @@
public abstract class LicenseMojo extends AbstractMojo {
+ private static final String SHADOWED_KEY = LicenseMojo.class.getName() + "_shadowed";
@Parameter
protected List<Override> overrides = new ArrayList<>();
@@ -249,7 +250,7 @@
});
}
- protected void addDependenciesToLicenseMap() throws ProjectBuildingException {
+ protected void addDependenciesToLicenseMap() throws ProjectBuildingException, MojoExecutionException, IOException {
Map<MavenProject, List<Pair<String, String>>> dependencyLicenseMap = gatherDependencies();
dependencyLicenseMap.forEach((depProject, value) -> {
Set<String> locations = dependencySets.isEmpty() ? Collections.singleton(location)
@@ -310,7 +311,9 @@
licenseUrl = fakeLicenseUrl;
}
}
- addProject(new Project(depProject, depLocation, depProject.getArtifact().getFile()),
+ addProject(
+ new Project(depProject, depLocation, depProject.getArtifact().getFile(),
+ Boolean.parseBoolean(String.valueOf(depProject.getContextValue(SHADOWED_KEY)))),
new LicenseSpec(licenseUrl, displayName), true);
}
@@ -363,11 +366,13 @@
}
}
- protected Map<MavenProject, List<Pair<String, String>>> gatherDependencies() throws ProjectBuildingException {
+ protected Map<MavenProject, List<Pair<String, String>>> gatherDependencies()
+ throws ProjectBuildingException, MojoExecutionException, IOException {
Map<MavenProject, List<Pair<String, String>>> dependencyLicenseMap = new HashMap<>();
Map<String, MavenProject> dependencyGavMap = new HashMap<>();
gatherProjectDependencies(project, dependencyLicenseMap, dependencyGavMap);
+ processExtraDependencies(dependencyLicenseMap, dependencyGavMap);
for (Override override : overrides) {
// Collect both <gav></gav> and <gavs><gav></gav><gav></gav>...</gavs>
@@ -399,27 +404,40 @@
return dependencyLicenseMap;
}
- private void gatherProjectDependencies(MavenProject project,
+ protected void gatherProjectDependencies(MavenProject project,
Map<MavenProject, List<Pair<String, String>>> dependencyLicenseMap,
- Map<String, MavenProject> dependencyGavMap) throws ProjectBuildingException {
+ Map<String, MavenProject> dependencyGavMap) throws ProjectBuildingException, MojoExecutionException {
final Set<Artifact> dependencyArtifacts = project.getArtifacts();
if (dependencyArtifacts != null) {
for (Artifact depArtifact : dependencyArtifacts) {
- processArtifact(depArtifact, dependencyLicenseMap, dependencyGavMap);
+ processArtifact(depArtifact, dependencyLicenseMap, dependencyGavMap, false);
}
}
- for (String gav : extraDependencies) {
- ArtifactHandler handler = new DefaultArtifactHandler("jar");
- String[] gavParts = StringUtils.split(gav, ':');
- Artifact manualDep = new DefaultArtifact(gavParts[0], gavParts[1], gavParts[2], Artifact.SCOPE_COMPILE,
- "jar", null, handler);
- processArtifact(manualDep, dependencyLicenseMap, dependencyGavMap);
- }
}
- private void processArtifact(Artifact depArtifact,
+ protected void processExtraDependencies(Map<MavenProject, List<Pair<String, String>>> dependencyLicenseMap,
+ Map<String, MavenProject> dependencyGavMap) throws ProjectBuildingException, MojoExecutionException {
+ for (String extraDependency : extraDependencies) {
+ String[] gavParts = StringUtils.split(extraDependency, ':');
+ String gav = gavParts[0] + ":" + gavParts[1] + ":" + gavParts[2];
+ if (!dependencyGavMap.containsKey(gav)) {
+ ArtifactHandler handler = new DefaultArtifactHandler("jar");
+ Artifact manualDep = new DefaultArtifact(gavParts[0], gavParts[1], gavParts[2], Artifact.SCOPE_COMPILE,
+ "jar", null, handler);
+ boolean shadowed = false;
+ if (gavParts.length > 3) {
+ shadowed = "shadowed".equals(gavParts[3]);
+ }
+ processArtifact(manualDep, dependencyLicenseMap, dependencyGavMap, shadowed);
+ } else {
+ getLog().warn("not adding extra dependency " + gav + " as it is already added");
+ }
+ }
+ }
+
+ protected void processArtifact(Artifact depArtifact,
Map<MavenProject, List<Pair<String, String>>> dependencyLicenseMap,
- Map<String, MavenProject> dependencyGavMap) throws ProjectBuildingException {
+ Map<String, MavenProject> dependencyGavMap, boolean shadowed) throws ProjectBuildingException {
if (!excludedScopes.contains(depArtifact.getScope())) {
MavenProject dep = resolveDependency(depArtifact);
if (!depArtifact.isResolved()) {
@@ -442,6 +460,7 @@
: (license1.getName() != null ? license1.getName() : "LICENSE_EMPTY_NAME_URL");
licenseUrls.add(new ImmutablePair<>(url, license1.getName()));
}
+ dep.setContextValue(SHADOWED_KEY, shadowed);
dependencyLicenseMap.put(dep, licenseUrls);
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/project/LicensedProjects.java b/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/project/LicensedProjects.java
index f6f9f32..31b4a86 100644
--- a/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/project/LicensedProjects.java
+++ b/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/project/LicensedProjects.java
@@ -21,6 +21,7 @@
import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;
+import java.util.function.Predicate;
import org.apache.hyracks.maven.license.LicenseSpec;
@@ -51,6 +52,11 @@
return projects;
}
+ public SortedSet<Project> getNonShadowedProjects() {
+ return projects.stream().filter(Predicate.not(Project::isShadowed))
+ .collect(() -> new TreeSet<>(Project.PROJECT_COMPARATOR), TreeSet::add, TreeSet::addAll);
+ }
+
public void addProject(Project project) {
projects.add(project);
}
diff --git a/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/project/Project.java b/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/project/Project.java
index e44914d..a755c05 100644
--- a/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/project/Project.java
+++ b/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/project/Project.java
@@ -41,6 +41,7 @@
private String licenseText;
private String sourcePointer;
private String classifier;
+ private final boolean shadowed;
@JsonIgnore
private MavenProject mavenProject;
@@ -48,7 +49,7 @@
public static final Comparator<Project> PROJECT_COMPARATOR =
(o1, o2) -> o1.compareToken().compareTo(o2.compareToken());
- public Project(MavenProject project, String location, File artifactPath) {
+ public Project(MavenProject project, String location, File artifactPath, boolean shadowed) {
mavenProject = project;
name = project.getName();
groupId = project.getGroupId();
@@ -57,6 +58,7 @@
url = project.getUrl();
classifier = project.getArtifact().getClassifier();
this.artifactPath = artifactPath.getPath();
+ this.shadowed = shadowed;
setLocation(location);
}
@@ -65,7 +67,8 @@
@JsonProperty("artifactId") String artifactId, @JsonProperty("url") String url,
@JsonProperty("version") String version, @JsonProperty("location") String location,
@JsonProperty("artifactPath") String artifactPath, @JsonProperty("noticeText") String noticeText,
- @JsonProperty("licenseText") String licenseText, @JsonProperty("classifier") String classifier) {
+ @JsonProperty("licenseText") String licenseText, @JsonProperty("classifier") String classifier,
+ @JsonProperty("shadowed") boolean shadowed) {
this.name = name;
this.groupId = groupId;
this.artifactId = artifactId;
@@ -76,6 +79,7 @@
this.noticeText = noticeText;
this.licenseText = licenseText;
this.classifier = classifier;
+ this.shadowed = shadowed;
}
public String getName() {
@@ -185,6 +189,10 @@
this.sourcePointer = sourcePointer;
}
+ public boolean isShadowed() {
+ return shadowed;
+ }
+
@Override
public String toString() {
return "Project [" + gav() + "]";