Skip to content
Open
Original file line number Diff line number Diff line change
Expand Up @@ -477,7 +477,8 @@ private Expression getConvertExpression(

case TIME:
return RexImpTable.optimize2(operand, Expressions.isConstantNull(format)
? Expressions.call(BuiltInMethod.UNIX_TIME_TO_STRING.method, operand)
? Expressions.call(BuiltInMethod.UNIX_TIME_TO_STRING_WITH_PRECISION.method,
operand, Expressions.constant(sourceType.getPrecision()))
: Expressions.call(
Expressions.new_(
BuiltInMethod.FORMAT_TIME.method.getDeclaringClass()),
Expand All @@ -494,7 +495,8 @@ private Expression getConvertExpression(

case TIMESTAMP:
return RexImpTable.optimize2(operand, Expressions.isConstantNull(format)
? Expressions.call(BuiltInMethod.UNIX_TIMESTAMP_TO_STRING.method, operand)
? Expressions.call(BuiltInMethod.UNIX_TIMESTAMP_TO_STRING_WITH_PRECISION.method,
operand, Expressions.constant(sourceType.getPrecision()))
: Expressions.call(
Expressions.new_(
BuiltInMethod.FORMAT_TIMESTAMP.method.getDeclaringClass()),
Expand Down
127 changes: 127 additions & 0 deletions core/src/main/java/org/apache/calcite/rex/RexBuilder.java
Original file line number Diff line number Diff line change
Expand Up @@ -29,17 +29,21 @@
import org.apache.calcite.rel.type.RelDataTypeSystemImpl;
import org.apache.calcite.runtime.FlatLists;
import org.apache.calcite.runtime.SqlFunctions;
import org.apache.calcite.sql.SqlAbstractDateTimeLiteral;
import org.apache.calcite.sql.SqlAggFunction;
import org.apache.calcite.sql.SqlCollation;
import org.apache.calcite.sql.SqlIntervalQualifier;
import org.apache.calcite.sql.SqlKind;
import org.apache.calcite.sql.SqlOperator;
import org.apache.calcite.sql.SqlSpecialOperator;
import org.apache.calcite.sql.SqlTimeLiteral;
import org.apache.calcite.sql.SqlTimestampLiteral;
import org.apache.calcite.sql.SqlUtil;
import org.apache.calcite.sql.fun.SqlCountAggFunction;
import org.apache.calcite.sql.fun.SqlLibraryOperators;
import org.apache.calcite.sql.fun.SqlStdOperatorTable;
import org.apache.calcite.sql.parser.SqlParserPos;
import org.apache.calcite.sql.parser.SqlParserUtil;
import org.apache.calcite.sql.type.ArraySqlType;
import org.apache.calcite.sql.type.IntervalSqlType;
import org.apache.calcite.sql.type.MapSqlType;
Expand Down Expand Up @@ -805,6 +809,12 @@ public RexNode makeCast(
&& SqlTypeUtil.isExactNumeric(type)) {
return makeCastBooleanToExact(type, exp);
}
final RexNode literalCast =
makeCastForTemporalLiteral(
pos, type, literal, matchNullability, safe, format);
if (literalCast != null) {
return literalCast;
}
if (canRemoveCastFromLiteral(type, value, typeName)) {
switch (typeName) {
case INTERVAL_YEAR:
Expand Down Expand Up @@ -877,6 +887,123 @@ public RexNode makeCast(
return makeAbstractCast(pos, type, exp, safe, format);
}

private @Nullable RexNode makeCastForTemporalLiteral(
SqlParserPos pos,
RelDataType type,
RexLiteral literal,
boolean matchNullability,
boolean safe,
RexLiteral format) {
if (!format.isNull()) {
return null;
}
if (SqlTypeUtil.isCharacter(literal.getType())) {
return makeCastFromCharacterLiteralToTemporal(
pos, type, literal, matchNullability, safe, format);
}
if (SqlTypeUtil.isCharacter(type)) {
return makeCharacterLiteralFromTemporalLiteral(type, literal);
}
return null;
}

private @Nullable RexNode makeCastFromCharacterLiteralToTemporal(
SqlParserPos pos,
RelDataType type,
RexLiteral literal,
boolean matchNullability,
boolean safe,
RexLiteral format) {
final NlsString nlsString = literal.getValueAs(NlsString.class);
if (nlsString == null) {
return null;
}
final String value = nlsString.getValue().trim();
final RexNode temporalLiteral;
try {
switch (type.getSqlTypeName()) {
case TIME:
final SqlTimeLiteral timeLiteral =
SqlParserUtil.parseTimeLiteral(value, pos);
if (!isExactFractionalSecondLiteral(timeLiteral, value)) {
return null;
}
final TimeString time =
requireNonNull(timeLiteral.getValueAs(TimeString.class),
"timeLiteral.getValueAs(TimeString.class)");
temporalLiteral = makeTimeLiteral(time, type.getPrecision());
break;
case TIMESTAMP:
final SqlTimestampLiteral timestampLiteral =
SqlParserUtil.parseTimestampLiteral(value, pos);
if (!isExactFractionalSecondLiteral(timestampLiteral, value)) {
return null;
}
final TimestampString timestamp =
requireNonNull(timestampLiteral.getValueAs(TimestampString.class),
"timestampLiteral.getValueAs(TimestampString.class)");
temporalLiteral = makeTimestampLiteral(timestamp, type.getPrecision());
break;
default:
return null;
}
} catch (RuntimeException e) {
return safe ? makeNullLiteral(type) : null;
}
if (type.isNullable()
&& !temporalLiteral.getType().isNullable()
&& matchNullability) {
return makeAbstractCast(pos, type, temporalLiteral, safe, format);
}
return temporalLiteral;
}

/** Converts a TIME or TIMESTAMP literal to a character literal.
*
* <p>Returns null if the literal is not a TIME or TIMESTAMP, the literal
* cannot be read as the corresponding temporal string value, or the formatted
* temporal value does not fit in the target character type. */
private @Nullable RexNode makeCharacterLiteralFromTemporalLiteral(
RelDataType type,
RexLiteral literal) {
// Format temporal literals directly so they do not go through generated
// code, whose TIME/TIMESTAMP runtime values have millisecond precision.
final String value;
final int precision = literal.getType().getPrecision();
switch (literal.getType().getSqlTypeName()) {
case TIME:
final TimeString time = literal.getValueAs(TimeString.class);
if (time == null) {
return null;
}
value = time.toString(precision);
break;
case TIMESTAMP:
final TimestampString timestamp = literal.getValueAs(TimestampString.class);
if (timestamp == null) {
return null;
}
value = timestamp.toString(precision);
break;
default:
return null;
}
// Only create a character literal if the formatted temporal value fits in
// the target type.
return SqlTypeUtil.comparePrecision(type.getPrecision(), value.length()) >= 0
Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

please add a comment explaining what happens here

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Added a note here about when this can replace the CAST with a literal.

? makeLiteral(value, type, true)
: null;
}

/** Returns whether a parsed TIME or TIMESTAMP literal has fractional-second
* precision high enough to preserve the original character value exactly. */
private static boolean isExactFractionalSecondLiteral(
Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

can you document what this function is supposed to do?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Added a short JavaDoc. This checks that the parsed literal has explicit fractional-second precision and still formats back to the original string.

SqlAbstractDateTimeLiteral literal, String value) {
return literal.getPrec() != 0
&& literal.getPrec() != RelDataType.PRECISION_NOT_SPECIFIED
&& literal.toFormattedString().equals(value);
}

/** Returns the lowest granularity unit for the given unit.
* YEAR and MONTH intervals are stored as months;
* HOUR, MINUTE, SECOND intervals are stored as milliseconds. */
Expand Down
27 changes: 24 additions & 3 deletions core/src/main/java/org/apache/calcite/rex/RexExecutorImpl.java
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@

import java.lang.reflect.Modifier;
import java.lang.reflect.Type;
import java.util.ArrayList;
import java.util.List;

/**
Expand Down Expand Up @@ -136,14 +137,34 @@ public static RexExecutable getExecutable(RexBuilder rexBuilder, List<RexNode> e
@Override public void reduce(RexBuilder rexBuilder, List<RexNode> constExps,
List<RexNode> reducedValues) {
assert reducedValues.isEmpty();
final List<RexNode> exps = new ArrayList<>();
Comment thread
mihaibudiu marked this conversation as resolved.
final List<Integer> ordinals = new ArrayList<>();
for (int i = 0; i < constExps.size(); i++) {
final RexNode constExp = constExps.get(i);
// Literals are already reduced. Keep them as Rex values instead of
// round-tripping through generated code.
if (!(constExp instanceof RexLiteral)) {
ordinals.add(i);
exps.add(constExp);
}
}
if (exps.isEmpty()) {
reducedValues.addAll(constExps);
return;
}
try {
String code = compile(rexBuilder, constExps, (list, index, storageType) -> {
String code = compile(rexBuilder, exps, (list, index, storageType) -> {
throw new UnsupportedOperationException();
});

final RexExecutable executable = new RexExecutable(code, constExps);
final RexExecutable executable = new RexExecutable(code, exps);
executable.setDataContext(dataContext);
executable.reduce(rexBuilder, constExps, reducedValues);
final List<RexNode> values = new ArrayList<>(exps.size());
executable.reduce(rexBuilder, exps, values);
reducedValues.addAll(constExps);
for (int i = 0; i < ordinals.size(); i++) {
reducedValues.set(ordinals.get(i), values.get(i));
}
} catch (RuntimeException ex) {
// Something went wrong during constant reduction (for example,
// we may have attempted a division by zero).
Expand Down
4 changes: 4 additions & 0 deletions core/src/main/java/org/apache/calcite/util/BuiltInMethod.java
Original file line number Diff line number Diff line change
Expand Up @@ -779,8 +779,12 @@ public enum BuiltInMethod {
String.class, int.class),
UNIX_DATE_TO_STRING(DateTimeUtils.class, "unixDateToString", int.class),
UNIX_TIME_TO_STRING(DateTimeUtils.class, "unixTimeToString", int.class),
UNIX_TIME_TO_STRING_WITH_PRECISION(DateTimeUtils.class, "unixTimeToString",
Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Both these methods assume that the actual data value is expressed as an integer number of milliseconds, so they won't work for precisions higher than 3. Perhaps this is not worsening the problem, but it looks to me like it will need to be fixed at some point again.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Right. I ran into this while debugging CalciteSqlOperatorTest.testConcatWSFunc. This is still limited by the runtime representation; a lot of the enumerable path uses milliseconds / Calendar internally, so this only preserves up to 3 fractional digits today.

int.class, int.class),
UNIX_TIMESTAMP_TO_STRING(DateTimeUtils.class, "unixTimestampToString",
long.class),
UNIX_TIMESTAMP_TO_STRING_WITH_PRECISION(DateTimeUtils.class,
"unixTimestampToString", long.class, int.class),
INTERVAL_YEAR_MONTH_TO_STRING(DateTimeUtils.class,
"intervalYearMonthToString", int.class, TimeUnitRange.class),
INTERVAL_DAY_TIME_TO_STRING(DateTimeUtils.class, "intervalDayTimeToString",
Expand Down
Loading
Loading