Skip to content

Commit

Permalink
Support generating sequences in the Faker connector
Browse files Browse the repository at this point in the history
  • Loading branch information
nineinchnick committed Dec 28, 2024
1 parent d192116 commit e958cd3
Show file tree
Hide file tree
Showing 9 changed files with 333 additions and 82 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -189,6 +189,13 @@ public static void add(long leftHigh, long leftLow, long rightHigh, long rightLo
}
}

public static Int128 add(Int128 left, Int128 right)
{
long[] result = new long[2];
add(left.getHigh(), left.getLow(), right.getHigh(), right.getLow(), result, 0);
return Int128.valueOf(result);
}

public static long addWithOverflow(long leftHigh, long leftLow, long rightHigh, long rightLow, long[] decimal, int offset)
{
long low = leftLow + rightLow;
Expand Down
3 changes: 3 additions & 0 deletions docs/src/main/sphinx/connector/faker.md
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,9 @@ The following table details all supported column properties.
* - `allowed_values`
- List of allowed values. Cannot be set together with the `min`, or `max`
properties.
* - `step`
- If set, generate sequential values with this step. For date and time columns
set this to a duration. Cannot be set for character-based type columns.
:::

### Character types
Expand Down
11 changes: 5 additions & 6 deletions plugin/trino-faker/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,11 @@
<artifactId>configuration</artifactId>
</dependency>

<dependency>
<groupId>io.airlift</groupId>
<artifactId>units</artifactId>
</dependency>

<dependency>
<groupId>io.trino</groupId>
<artifactId>trino-main</artifactId>
Expand Down Expand Up @@ -119,12 +124,6 @@
<scope>runtime</scope>
</dependency>

<dependency>
<groupId>io.airlift</groupId>
<artifactId>units</artifactId>
<scope>runtime</scope>
</dependency>

<dependency>
<groupId>io.trino</groupId>
<artifactId>trino-client</artifactId>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ public record ColumnInfo(FakerColumnHandle handle, ColumnMetadata metadata)
public static final String MIN_PROPERTY = "min";
public static final String MAX_PROPERTY = "max";
public static final String ALLOWED_VALUES_PROPERTY = "allowed_values";
public static final String STEP_PROPERTY = "step";

public ColumnInfo
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,27 +15,38 @@
package io.trino.plugin.faker;

import com.google.common.collect.ImmutableList;
import io.airlift.units.Duration;
import io.trino.spi.TrinoException;
import io.trino.spi.connector.ColumnHandle;
import io.trino.spi.connector.ColumnMetadata;
import io.trino.spi.predicate.Domain;
import io.trino.spi.predicate.Range;
import io.trino.spi.predicate.ValueSet;
import io.trino.spi.type.CharType;
import io.trino.spi.type.TimeType;
import io.trino.spi.type.TimeWithTimeZoneType;
import io.trino.spi.type.TimestampType;
import io.trino.spi.type.TimestampWithTimeZoneType;
import io.trino.spi.type.Type;
import io.trino.spi.type.VarbinaryType;
import io.trino.spi.type.VarcharType;

import java.util.Collection;
import java.util.List;
import java.util.concurrent.TimeUnit;

import static com.google.common.base.Preconditions.checkState;
import static com.google.common.collect.ImmutableList.toImmutableList;
import static io.trino.plugin.faker.ColumnInfo.ALLOWED_VALUES_PROPERTY;
import static io.trino.plugin.faker.ColumnInfo.GENERATOR_PROPERTY;
import static io.trino.plugin.faker.ColumnInfo.MAX_PROPERTY;
import static io.trino.plugin.faker.ColumnInfo.MIN_PROPERTY;
import static io.trino.plugin.faker.ColumnInfo.NULL_PROBABILITY_PROPERTY;
import static io.trino.plugin.faker.ColumnInfo.STEP_PROPERTY;
import static io.trino.spi.StandardErrorCode.INVALID_COLUMN_PROPERTY;
import static io.trino.spi.type.BigintType.BIGINT;
import static io.trino.spi.type.DateType.DATE;
import static io.trino.spi.type.IntegerType.INTEGER;
import static java.util.Objects.requireNonNull;

public record FakerColumnHandle(
Expand All @@ -44,13 +55,17 @@ public record FakerColumnHandle(
Type type,
double nullProbability,
String generator,
Domain domain)
Domain domain,
ValueSet step)
implements ColumnHandle
{
public FakerColumnHandle
{
requireNonNull(name, "name is null");
requireNonNull(type, "type is null");
requireNonNull(domain, "domain is null");
requireNonNull(step, "step is null");
checkState(step.isNone() || step.isSingleValue(), "step must be a single value");
}

public static FakerColumnHandle of(int columnId, ColumnMetadata column, double defaultNullProbability)
Expand All @@ -63,20 +78,8 @@ public static FakerColumnHandle of(int columnId, ColumnMetadata column, double d
if (generator != null && !isCharacterColumn(column)) {
throw new TrinoException(INVALID_COLUMN_PROPERTY, "The `%s` property can only be set for CHAR, VARCHAR or VARBINARY columns".formatted(GENERATOR_PROPERTY));
}
Object min;
try {
min = Literal.parse((String) column.getProperties().get(MIN_PROPERTY), column.getType());
}
catch (IllegalArgumentException e) {
throw new TrinoException(INVALID_COLUMN_PROPERTY, "The `%s` property must be a valid %s literal".formatted(MIN_PROPERTY, column.getType().getDisplayName()), e);
}
Object max;
try {
max = Literal.parse((String) column.getProperties().get(MAX_PROPERTY), column.getType());
}
catch (IllegalArgumentException e) {
throw new TrinoException(INVALID_COLUMN_PROPERTY, "The `%s` property must be a valid %s literal".formatted(MAX_PROPERTY, column.getType().getDisplayName()), e);
}
Object min = propertyValue(column, MIN_PROPERTY);
Object max = propertyValue(column, MAX_PROPERTY);
Domain domain = Domain.all(column.getType());
if (min != null || max != null) {
if (isCharacterColumn(column)) {
Expand Down Expand Up @@ -106,14 +109,51 @@ public static FakerColumnHandle of(int columnId, ColumnMetadata column, double d
column.getType(),
nullProbability,
generator,
domain);
domain,
stepValue(column));
}

private static boolean isCharacterColumn(ColumnMetadata column)
{
return column.getType() instanceof CharType || column.getType() instanceof VarcharType || column.getType() instanceof VarbinaryType;
}

private static Object propertyValue(ColumnMetadata column, String property)
{
try {
return Literal.parse((String) column.getProperties().get(property), column.getType());
}
catch (IllegalArgumentException e) {
throw new TrinoException(INVALID_COLUMN_PROPERTY, "The `%s` property must be a valid %s literal".formatted(property, column.getType().getDisplayName()), e);
}
}

private static ValueSet stepValue(ColumnMetadata column)
{
Type type = column.getType();
String rawStep = (String) column.getProperties().get(STEP_PROPERTY);
if (rawStep == null) {
return ValueSet.none(type);
}
if (isCharacterColumn(column)) {
throw new TrinoException(INVALID_COLUMN_PROPERTY, "The `%s` property cannot be set for CHAR, VARCHAR or VARBINARY columns".formatted(STEP_PROPERTY));
}
if (DATE.equals(column.getType()) || type instanceof TimestampType || type instanceof TimestampWithTimeZoneType || type instanceof TimeType || type instanceof TimeWithTimeZoneType) {
try {
return ValueSet.of(BIGINT, Duration.valueOf(rawStep).roundTo(TimeUnit.NANOSECONDS));
}
catch (IllegalArgumentException e) {
throw new TrinoException(INVALID_COLUMN_PROPERTY, "The `%s` property for a %s column must be a valid duration literal".formatted(STEP_PROPERTY, column.getType().getDisplayName()), e);
}
}
try {
return ValueSet.of(type, Literal.parse(rawStep, type));
}
catch (IllegalArgumentException e) {
throw new TrinoException(INVALID_COLUMN_PROPERTY, "The `%s` property for a %s column must be a valid %s literal".formatted(STEP_PROPERTY, column.getType().getDisplayName(), type.getDisplayName()), e);
}
}

private static Range range(Type type, Object min, Object max)
{
requireNonNull(type, "type is null");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@
import static io.trino.plugin.faker.ColumnInfo.ALLOWED_VALUES_PROPERTY;
import static io.trino.plugin.faker.ColumnInfo.MAX_PROPERTY;
import static io.trino.plugin.faker.ColumnInfo.MIN_PROPERTY;
import static io.trino.plugin.faker.ColumnInfo.STEP_PROPERTY;
import static io.trino.spi.StandardErrorCode.INVALID_COLUMN_PROPERTY;
import static io.trino.spi.StandardErrorCode.INVALID_SCHEMA_PROPERTY;
import static io.trino.spi.StandardErrorCode.INVALID_TABLE_PROPERTY;
Expand Down Expand Up @@ -188,7 +189,12 @@ public List<PropertyMetadata<?>> getColumnProperties()
value -> ((List<?>) value).stream()
.map(String.class::cast)
.collect(toImmutableList()),
value -> value));
value -> value),
stringProperty(
STEP_PROPERTY,
"If set, generate sequential values with this step. For date and time columns set this to a duration",
null,
false));
}

private static void checkProperty(boolean expression, ErrorCodeSupplier errorCode, String errorMessage)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,8 @@
import io.trino.spi.function.FunctionMetadata;
import io.trino.spi.function.SchemaFunctionName;
import io.trino.spi.predicate.Domain;
import io.trino.spi.predicate.Range;
import io.trino.spi.predicate.ValueSet;
import io.trino.spi.security.TrinoPrincipal;
import io.trino.spi.statistics.ComputedStatistics;
import io.trino.spi.type.BigintType;
Expand Down Expand Up @@ -336,7 +338,8 @@ public synchronized FakerOutputTableHandle beginCreateTable(ConnectorSession ses
BigintType.BIGINT,
0,
"",
Domain.all(BigintType.BIGINT)),
Domain.create(ValueSet.ofRanges(Range.greaterThanOrEqual(BigintType.BIGINT, 0L)), false),
ValueSet.of(BigintType.BIGINT, 1L)),
ColumnMetadata.builder()
.setName(ROW_ID_COLUMN_NAME)
.setType(BigintType.BIGINT)
Expand Down
Loading

0 comments on commit e958cd3

Please sign in to comment.