Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

update datagen so iteration works properly and add schemas to use for testing #653

Merged
merged 1 commit into from
Jan 29, 2018
Merged

update datagen so iteration works properly and add schemas to use for testing #653

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
93 changes: 58 additions & 35 deletions ksql-examples/src/main/java/io/confluent/avro/random/generator/Generator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ public class Generator {
private static final Schema.Parser schemaParser = new Schema.Parser();
private static final Map<Schema, Generex> generexCache = new HashMap<>();
private static final Map<Schema, List<Object>> optionsCache = new HashMap<>();
private static final Map<Schema, Iterator<Object>> iteratorCache = new HashMap<>();
private static final Map<String, Iterator<Object>> iteratorCache = new HashMap<>();

/**
* The name to use for the top-level JSON property when specifying ARG-specific attributes.
Expand Down Expand Up @@ -276,16 +276,16 @@ public Schema schema() {
* </table>
*/
public Object generate() {
return generateObject(topLevelSchema);
return generateObject(topLevelSchema, "");
}

private Object generateObject(Schema schema) {
private Object generateObject(Schema schema, String fieldName) {
Map propertiesProp = getProperties(schema).orElse(Collections.emptyMap());
if (propertiesProp.containsKey(OPTIONS_PROP)) {
return generateOption(schema, propertiesProp);
}
if (propertiesProp.containsKey(ITERATION_PROP)) {
return generateIteration(schema, propertiesProp);
return generateIteration(schema, propertiesProp, fieldName);
}
switch (schema.getType()) {
case ARRAY:
Expand Down Expand Up @@ -783,27 +783,7 @@ private Iterator<Object> parseIterations(Schema schema, Map propertiesProp) {
case BOOLEAN:
return getBooleanIterator(iterationProps);
case INT: {
Integer iterationStartField = getIntegerNumberField(
ITERATION_PROP,
ITERATION_PROP_START,
iterationProps
);
Integer iterationRestartField = getIntegerNumberField(
ITERATION_PROP,
ITERATION_PROP_RESTART,
iterationProps
);
Integer iterationStepField = getIntegerNumberField(
ITERATION_PROP,
ITERATION_PROP_STEP,
iterationProps
);
return getIntegralIterator(
iterationStartField != null ? iterationStartField.longValue() : null,
iterationRestartField != null ? iterationRestartField.longValue() : null,
iterationStepField != null ? iterationStepField.longValue() : null,
IntegralIterator.Type.INTEGER
);
return createIntegerIterator(iterationProps);
}
case LONG: {
Long iterationStartField = getIntegralNumberField(
Expand Down Expand Up @@ -874,6 +854,9 @@ private Iterator<Object> parseIterations(Schema schema, Map propertiesProp) {
DecimalIterator.Type.DOUBLE
);
}
case STRING:
return createStringIterator(createIntegerIterator(iterationProps));

default:
throw new UnsupportedOperationException(String.format(
"%s property can only be specified on numeric and boolean schemas, not %s schema",
Expand All @@ -890,19 +873,59 @@ private Iterator<Object> parseIterations(Schema schema, Map propertiesProp) {
}
}

private Iterator<Object> createStringIterator(Iterator<Object> inner) {
return new Iterator<Object>() {
@Override
public boolean hasNext() {
return inner.hasNext();
}

@Override
public Object next() {
return inner.next().toString();
}
};
}

private Iterator<Object> createIntegerIterator(Map iterationProps) {
Integer iterationStartField = getIntegerNumberField(
ITERATION_PROP,
ITERATION_PROP_START,
iterationProps
);
Integer iterationRestartField = getIntegerNumberField(
ITERATION_PROP,
ITERATION_PROP_RESTART,
iterationProps
);
Integer iterationStepField = getIntegerNumberField(
ITERATION_PROP,
ITERATION_PROP_STEP,
iterationProps
);
return getIntegralIterator(
iterationStartField != null ? iterationStartField.longValue() : null,
iterationRestartField != null ? iterationRestartField.longValue() : null,
iterationStepField != null ? iterationStepField.longValue() : null,
IntegralIterator.Type.INTEGER
);
}

@SuppressWarnings("unchecked")
private <T> T generateIteration(Schema schema, Map propertiesProp) {
if (!iteratorCache.containsKey(schema)) {
iteratorCache.put(schema, parseIterations(schema, propertiesProp));
private <T> T generateIteration(Schema schema,
Map propertiesProp,
String fieldName) {
if (!iteratorCache.containsKey(fieldName)) {
iteratorCache.put(fieldName, parseIterations(schema, propertiesProp));
}
return (T) iteratorCache.get(schema).next();
return (T) iteratorCache.get(fieldName).next();
}

private Collection<Object> generateArray(Schema schema, Map propertiesProp) {
int length = getLengthBounds(propertiesProp).random();
Collection<Object> result = new ArrayList<>(length);
for (int i = 0; i < length; i++) {
result.add(generateObject(schema.getElementType()));
result.add(generateObject(schema.getElementType(), ""));
}
return result;
}
Expand Down Expand Up @@ -1051,7 +1074,7 @@ private Map<String, Object> generateMap(Schema schema, Map propertiesProp) {
Object keyProp = propertiesProp.get(KEYS_PROP);
if (keyProp == null) {
for (int i = 0; i < length; i++) {
result.put(generateRandomString(1), generateObject(schema.getValueType()));
result.put(generateRandomString(1), generateObject(schema.getValueType(), ""));
}
} else if (keyProp instanceof Map) {
Map keyPropMap = (Map) keyProp;
Expand All @@ -1060,14 +1083,14 @@ private Map<String, Object> generateMap(Schema schema, Map propertiesProp) {
optionsCache.put(schema, parseOptions(Schema.create(Schema.Type.STRING), keyPropMap));
}
for (int i = 0; i < length; i++) {
result.put(generateOption(schema, keyPropMap), generateObject(schema.getValueType()));
result.put(generateOption(schema, keyPropMap), generateObject(schema.getValueType(), ""));
}
} else {
int keyLength = getLengthBounds(keyPropMap.get(LENGTH_PROP)).random();
for (int i = 0; i < length; i++) {
result.put(
generateRandomString(keyLength),
generateObject(schema.getValueType())
generateObject(schema.getValueType(), "")
);
}
}
Expand All @@ -1087,7 +1110,7 @@ private Object generateNull() {
private GenericRecord generateRecord(Schema schema) {
GenericRecordBuilder builder = new GenericRecordBuilder(schema);
for (Schema.Field field : schema.getFields()) {
builder.set(field, generateObject(field.schema()));
builder.set(field, generateObject(field.schema(), field.name()));
}
return builder.build();
}
Expand Down Expand Up @@ -1124,7 +1147,7 @@ private String generateString(Schema schema, Map propertiesProp) {

private Object generateUnion(Schema schema) {
List<Schema> schemas = schema.getTypes();
return generateObject(schemas.get(random.nextInt(schemas.size())));
return generateObject(schemas.get(random.nextInt(schemas.size())), "");
}

private LengthBounds getLengthBounds(Map propertiesProp) {
Expand Down
31 changes: 31 additions & 0 deletions ksql-examples/src/main/resources/inventory.avro
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
{
"namespace": "ksql",
"name": "inventory",
"type": "record",
"fields": [
{"name": "id", "type": {
"type": "long",
"arg.properties": {
"iteration": {
"start": 0
}
}
}},
{"name": "quantity", "type": {
"type": "long",
"arg.properties": {
"iteration": {
"start": 0
}
}
}},
{"name": "productid", "type": {
"type": "long",
"arg.properties": {
"iteration": {
"start": 0
}
}
}}
]
}
39 changes: 39 additions & 0 deletions ksql-examples/src/main/resources/product.avro
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
{
"namespace": "ksql",
"name": "product",
"type": "record",
"fields": [
{"name": "id", "type": {
"type": "long",
"arg.properties": {
"iteration": {
"start": 0
}
}
}},
{"name": "name", "type": {
"type": "string",
"arg.properties": {
"iteration": {
"start": 0
}
}
}},
{"name": "description", "type": {
"type": "string",
"arg.properties": {
"iteration": {
"start": 0
}
}
}},
{"name": "price", "type": {
"type": "double",
"arg.properties": {
"iteration": {
"start": 0
}
}
}}
]
}
39 changes: 39 additions & 0 deletions ksql-examples/src/main/resources/purchase.avro
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
{
"namespace": "ksql",
"name": "purchase",
"type": "record",
"fields": [
{"name": "id", "type": {
"type": "long",
"arg.properties": {
"iteration": {
"start": 0
}
}
}},
{"name": "quantity", "type": {
"type": "long",
"arg.properties": {
"iteration": {
"start": 0
}
}
}},
{"name": "productid", "type": {
"type": "long",
"arg.properties": {
"iteration": {
"start": 0
}
}
}},
{"name": "total", "type": {
"type": "double",
"arg.properties": {
"iteration": {
"start": 0
}
}
}}
]
}