diff options
Diffstat (limited to 'lib/java/src/org/apache/thrift/partial')
9 files changed, 2015 insertions, 0 deletions
diff --git a/lib/java/src/org/apache/thrift/partial/EnumCache.java b/lib/java/src/org/apache/thrift/partial/EnumCache.java new file mode 100644 index 000000000..22423f10c --- /dev/null +++ b/lib/java/src/org/apache/thrift/partial/EnumCache.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.partial; + +import org.apache.thrift.partial.Validate; + +import org.apache.thrift.TEnum; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.HashMap; +import java.util.Map; + +/** + * Provides a memoized way to lookup an enum by its value. + * + * This class is used internally by {@code TDeserializer}. + * It is not intended to be used separately on its own. + */ +public class EnumCache { + private static Logger LOG = LoggerFactory.getLogger(EnumCache.class); + + private Map<Class<? extends TEnum>, Map<Integer, TEnum>> classMap; + + public EnumCache() { + this.classMap = new HashMap<>(); + } + + /** + * Gets an instance of the enum type {@code enumClass} + * corresponding to the given {@code value}. + * + * @param enumClass class of the enum to be returned. + * @param value value returned by {@code getValue()}. + */ + public TEnum get(Class<? extends TEnum> enumClass, int value) { + Validate.checkNotNull(enumClass, "enumClass"); + + Map<Integer, TEnum> valueMap = classMap.get(enumClass); + if (valueMap == null) { + valueMap = addClass(enumClass); + if (valueMap == null) { + return null; + } + } + + return valueMap.get(value); + } + + private Map<Integer, TEnum> addClass(Class<? extends TEnum> enumClass) { + try { + Method valuesMethod = enumClass.getMethod("values"); + TEnum[] enumValues = (TEnum[]) valuesMethod.invoke(null); + Map<Integer, TEnum> valueMap = new HashMap<>(); + + for (TEnum enumValue : enumValues) { + valueMap.put(enumValue.getValue(), enumValue); + } + + classMap.put(enumClass, valueMap); + return valueMap; + } catch (NoSuchMethodException e) { + LOG.error("enum class does not have values() method", e); + return null; + } catch (IllegalAccessException e) { + LOG.error("Enum.values() method should be public!", e); + return null; + } catch (InvocationTargetException e) { + LOG.error("Enum.values() threw exception", e); + return null; + } + } +} diff --git a/lib/java/src/org/apache/thrift/partial/PartialThriftComparer.java b/lib/java/src/org/apache/thrift/partial/PartialThriftComparer.java new file mode 100644 index 000000000..f0f33eb4f --- /dev/null +++ b/lib/java/src/org/apache/thrift/partial/PartialThriftComparer.java @@ -0,0 +1,376 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.partial; + +import org.apache.thrift.TBase; +import org.apache.thrift.protocol.TType; + +import java.lang.StringBuilder; +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Enables comparison of two TBase instances such that the comparison + * is limited to the subset of fields defined by the supplied metadata. + * + * This comparer is useful when comparing two instances where: + * -- one is generated by full deserialization. + * -- the other is generated by partial deserialization. + * + * The typical use case is to establish correctness of partial deserialization. + */ +public class PartialThriftComparer<T extends TBase> { + + private enum ComparisonResult { + UNKNOWN, + EQUAL, + NOT_EQUAL + } + + // Metadata that defines the scope of comparison. + private ThriftMetadata.ThriftStruct metadata; + + /** + * Constructs an instance of {@link PartialThriftComparer}. + * + * @param metadata defines the scope of comparison. + */ + public PartialThriftComparer(ThriftMetadata.ThriftStruct metadata) { + this.metadata = metadata; + } + + /** + * Compares thrift objects {@code t1} and {@code t2} and + * returns true if they are equal false otherwise. The comparison is limited + * to the scope defined by {@code metadata}. + * <p> + * If the objects are not equal then it optionally records their differences + * if {@code sb} is supplied. + * <p> + * + * @param t1 the first object. + * @param t2 the second object. + * @param sb if non-null, results of the comparison are returned in it. + * @return true if objects are equivalent, false otherwise. + */ + public boolean areEqual(T t1, T t2, StringBuilder sb) { + return this.areEqual(this.metadata, t1, t2, sb); + } + + private boolean areEqual( + ThriftMetadata.ThriftObject data, + Object o1, + Object o2, + StringBuilder sb) { + + byte fieldType = data.data.valueMetaData.type; + switch (fieldType) { + case TType.STRUCT: + return this.areEqual((ThriftMetadata.ThriftStruct) data, o1, o2, sb); + + case TType.LIST: + return this.areEqual((ThriftMetadata.ThriftList) data, o1, o2, sb); + + case TType.MAP: + return this.areEqual((ThriftMetadata.ThriftMap) data, o1, o2, sb); + + case TType.SET: + return this.areEqual((ThriftMetadata.ThriftSet) data, o1, o2, sb); + + case TType.ENUM: + return this.areEqual((ThriftMetadata.ThriftEnum) data, o1, o2, sb); + + case TType.BOOL: + case TType.BYTE: + case TType.I16: + case TType.I32: + case TType.I64: + case TType.DOUBLE: + case TType.STRING: + return this.areEqual((ThriftMetadata.ThriftPrimitive) data, o1, o2, sb); + + default: + throw unsupportedFieldTypeException(fieldType); + } + } + + private boolean areEqual( + ThriftMetadata.ThriftStruct data, + Object o1, + Object o2, + StringBuilder sb) { + ComparisonResult result = checkNullEquality(data, o1, o2, sb); + if (result != ComparisonResult.UNKNOWN) { + return result == ComparisonResult.EQUAL; + } + + TBase t1 = (TBase) o1; + TBase t2 = (TBase) o2; + + if (data.fields.size() == 0) { + if (t1.equals(t2)) { + return true; + } else { + appendNotEqual(data, sb, t1, t2, "struct1", "struct2"); + return false; + } + } else { + + boolean overallResult = true; + + for (Object o : data.fields.values()) { + ThriftMetadata.ThriftObject field = (ThriftMetadata.ThriftObject) o; + Object f1 = t1.getFieldValue(field.fieldId); + Object f2 = t2.getFieldValue(field.fieldId); + overallResult = overallResult && this.areEqual(field, f1, f2, sb); + } + + return overallResult; + } + } + + private boolean areEqual( + ThriftMetadata.ThriftPrimitive data, + Object o1, + Object o2, + StringBuilder sb) { + + ComparisonResult result = checkNullEquality(data, o1, o2, sb); + if (result != ComparisonResult.UNKNOWN) { + return result == ComparisonResult.EQUAL; + } + + if (data.isBinary()) { + if (areBinaryFieldsEqual(o1, o2)) { + return true; + } + } else if (o1.equals(o2)) { + return true; + } + + appendNotEqual(data, sb, o1, o2, "o1", "o2"); + return false; + } + + private boolean areEqual( + ThriftMetadata.ThriftEnum data, + Object o1, + Object o2, + StringBuilder sb) { + + ComparisonResult result = checkNullEquality(data, o1, o2, sb); + if (result != ComparisonResult.UNKNOWN) { + return result == ComparisonResult.EQUAL; + } + + if (o1.equals(o2)) { + return true; + } + + appendNotEqual(data, sb, o1, o2, "o1", "o2"); + return false; + } + + private boolean areEqual( + ThriftMetadata.ThriftList data, + Object o1, + Object o2, + StringBuilder sb) { + + List<Object> l1 = (List<Object>) o1; + List<Object> l2 = (List<Object>) o2; + + ComparisonResult result = checkNullEquality(data, o1, o2, sb); + if (result != ComparisonResult.UNKNOWN) { + return result == ComparisonResult.EQUAL; + } + + if (!checkSizeEquality(data, l1, l2, sb, "list")) { + return false; + } + + for (int i = 0; i < l1.size(); i++) { + Object e1 = l1.get(i); + Object e2 = l2.get(i); + if (!this.areEqual(data.elementData, e1, e2, sb)) { + return false; + } + } + + return true; + } + + private boolean areEqual( + ThriftMetadata.ThriftSet data, + Object o1, + Object o2, + StringBuilder sb) { + + Set<Object> s1 = (Set<Object>) o1; + Set<Object> s2 = (Set<Object>) o2; + + ComparisonResult result = checkNullEquality(data, o1, o2, sb); + if (result != ComparisonResult.UNKNOWN) { + return result == ComparisonResult.EQUAL; + } + + if (!checkSizeEquality(data, s1, s2, sb, "set")) { + return false; + } + + for (Object e1 : s1) { + if (!s2.contains(e1)) { + appendResult(data, sb, "Element %s in s1 not found in s2", e1); + return false; + } + } + + return true; + } + + private boolean areEqual( + ThriftMetadata.ThriftMap data, + Object o1, + Object o2, + StringBuilder sb) { + + Map<Object, Object> m1 = (Map<Object, Object>) o1; + Map<Object, Object> m2 = (Map<Object, Object>) o2; + + ComparisonResult result = checkNullEquality(data, o1, o2, sb); + if (result != ComparisonResult.UNKNOWN) { + return result == ComparisonResult.EQUAL; + } + + if (!checkSizeEquality(data, m1.keySet(), m2.keySet(), sb, "map.keySet")) { + return false; + } + + for (Object k1 : m1.keySet()) { + if (!m2.containsKey(k1)) { + appendResult(data, sb, "Key %s in m1 not found in m2", k1); + return false; + } + + Object v1 = m1.get(k1); + Object v2 = m2.get(k1); + if (!this.areEqual(data.valueData, v1, v2, sb)) { + return false; + } + } + + return true; + } + + private boolean areBinaryFieldsEqual(Object o1, Object o2) { + if (o1 instanceof byte[]) { + if (Arrays.equals((byte[]) o1, (byte[]) o2)) { + return true; + } + } else if (o1 instanceof ByteBuffer) { + if (((ByteBuffer) o1).compareTo((ByteBuffer) o2) == 0) { + return true; + } + } else { + throw new UnsupportedOperationException( + String.format("Unsupported binary field type: %s", o1.getClass().getName())); + } + + return false; + } + + private void appendResult( + ThriftMetadata.ThriftObject data, + StringBuilder sb, + String format, + Object... args) { + if (sb != null) { + String msg = String.format(format, args); + sb.append(data.fieldId.getFieldName()); + sb.append(" : "); + sb.append(msg); + } + } + + private void appendNotEqual( + ThriftMetadata.ThriftObject data, + StringBuilder sb, + Object o1, + Object o2, + String o1name, + String o2name) { + + String o1s = o1.toString(); + String o2s = o2.toString(); + + if ((o1s.length() + o2s.length()) < 100) { + appendResult(data, sb, "%s (%s) != %s (%s)", o1name, o1s, o2name, o2s); + } else { + appendResult( + data, sb, "%s != %s\n%s =\n%s\n%s =\n%s\n", + o1name, o2name, o1name, o1s, o2name, o2s); + } + } + + private ComparisonResult checkNullEquality( + ThriftMetadata.ThriftObject data, + Object o1, + Object o2, + StringBuilder sb) { + if ((o1 == null) && (o2 == null)) { + return ComparisonResult.EQUAL; + } + + if (o1 == null) { + appendResult(data, sb, "o1 (null) != o2"); + } + + if (o2 == null) { + appendResult(data, sb, "o1 != o2 (null)"); + } + + return ComparisonResult.UNKNOWN; + } + + private boolean checkSizeEquality( + ThriftMetadata.ThriftObject data, + Collection c1, + Collection c2, + StringBuilder sb, + String typeName) { + + if (c1.size() != c2.size()) { + appendResult( + data, sb, "%s1.size(%d) != %s2.size(%d)", + typeName, c1.size(), typeName, c2.size()); + return false; + } + + return true; + } + + static UnsupportedOperationException unsupportedFieldTypeException(byte fieldType) { + return new UnsupportedOperationException("field type not supported: '" + fieldType + "'"); + } +} diff --git a/lib/java/src/org/apache/thrift/partial/README.md b/lib/java/src/org/apache/thrift/partial/README.md new file mode 100644 index 000000000..d5794fae7 --- /dev/null +++ b/lib/java/src/org/apache/thrift/partial/README.md @@ -0,0 +1,112 @@ +# Partial Thrift Deserialization + +## Overview +This document describes how partial deserialization of Thrift works. There are two main goals of this documentation: +1. Make it easier to understand the current Java implementation in this folder. +1. Be useful in implementing partial deserialization support in additional languages. + +This document is divided into two high level areas. The first part explains important concepts relevant to partial deserialization. The second part describes components involved in the Java implementation in this folder. + +Moreover, this blog provides some performance numbers and addtional information: https://medium.com/pinterest-engineering/improving-data-processing-efficiency-using-partial-deserialization-of-thrift-16bc3a4a38b4 + +## Basic Concepts + +### Motivation + +The main motivation behind implementing this feature is to improve performance when we need to access only a subset of fields in any Thrift object. This situation arises often when big data is stored in Thrift encoded format (for example, SequenceFile with serialized Thrift values). Many data processing jobs may access this data. However, not every job needs to access every field of each object. In such cases, if we have prior knowledge of the fields needed for a given job, we can deserialize only that subset of fields and avoid the cost deserializing the rest of the fields. There are two benefits of this approach: we save cpu cycles by not deserializing unnecessary field and we end up reducing gc pressure. Both of the savings quickly add up when processing billions of instances in a data processing job. + +### Partial deserialization + +Partial deserialization involves deserializing only a subset of the fields of a serialized Thrift object while efficiently skipping over the rest. One very important benefit of partial deserialization is that the output of the deserialization process is not limited to a `TBase` derived object. It can deserialize a serialized blob into any type by using an appropriate `ThriftFieldValueProcessor`. + +### Defining the subset of fields to deserialize + +The subset of fields to deserialize is defined using a list of fully qualified field names. For example, consider the Thrift `struct` definition below: + +```Thrift +struct SmallStruct { + 1: optional string stringValue; + 2: optional i16 i16Value; +} + +struct TestStruct { + 1: optional i16 i16Field; + 2: optional list<SmallStruct> structList; + 3: optional set<SmallStruct> structSet; + 4: optional map<string, SmallStruct> structMap; + 5: optional SmallStruct structField; +} +``` + +For the Thrift `struct`, each of the following line shows a fully qualified field definition. Partial deserialization uses a non-empty set of such field definitions to identify the subset of fields to deserialize. + +``` +- i16Field +- structList.stringValue +- structSet.i16Value +- structMap.stringValue +- structField.i16Value +``` + +Note that the syntax of denoting paths involving map fields do not support a way to define sub-fields of the key type. + +For example, the field path `structMap.stringValue` shown above has leaf segment `stringValue` which is a field in map values. + +## Components + +The process of partial deserialization involves the following major components. We have listed names of the Java file(s) implementing each component for easier mapping to the source code. + +### Thrift Metadata + +Source files: +- ThriftField.java +- ThriftMetadata.java + +We saw in the previous section how we can identify the subset of fields to deserialize. As the first step, we need to compile the collection of field definitions into an efficient data structure that we can traverse at runtime. This step is achieved using `ThriftField` and `ThriftMetadata` classes. For example, + +```Java +// First, create a collection of fully qualified field names. +List<String> fieldNames = Arrays.asList("i16Field", "structField.i16Value"); + +// Convert the flat collection into an n-ary tree of fields. +List<ThriftField> fields = ThriftField.fromNames(fieldNames); + +// Compile the tree of fields into internally used metadata. +ThriftMetadata.ThriftStruct metadata = + ThriftMetadata.ThriftStruct.fromFields(TestStruct.class, fields); +``` + +At this point, we have an efficient internal representation of the fields that need to get deserialized. + +### Partial Thrift Protocol + +Source files: +- PartialThriftProtocol.java +- PartialThriftBinaryProtocol.java +- PartialThriftCompactProtocol.java + +This component implements efficient skipping over fields that need not be deserialized. Note that this skipping is more efficient compared to that achieved by using `TProtocolUtil.skip()`. The latter calls the corresponding `read()`, allocates and initializes certain values (for example, strings) and then discards the returned value. In comparison, `PartialThriftProtocol` skips a field by incrementing internal offset into the transport buffer. + +### Partial Thrift Deserializer + +Source files: +- PartialThriftDeserializer.java + +This component, traverses a serialized blob sequentially one field at a time. At the beginning of each field, it consults the informations stored in `ThriftMetadata` to see if that field needs to be deserialized. If yes, then the field is deserialized into a value as would normally take place during regular deserialization process. If that field is not in the target subset then the deserializer calls `PartialThriftProtocol` to efficiently skip over that field. + +### Field Value Processor + +Source files: +- ThriftFieldValueProcessor.java +- ThriftStructProcessor.java + +One very important benefit of partial deserialization is that the output of the deserialization process is not limited to a `TBase` derived object. It can deserialize a serialized blob into any type by using an appropriate `ThriftFieldValueProcessor`. + +When the partial Thrift deserializer deserializes a field, it passes its value to a `ThriftFieldValueProcessor`. The processor gets to decide whether the value is stored as-is or is stored in some intermediate form. The default implementation of this interface is `ThriftStructProcessor`. This implementation outputs a `TBase` derived object. There are other implementations that exist (not included in this drop at present). For example, one implementation enables deserializing a Thrift blob directly into an `InternalRow` used by `Spark`. That has yielded orders of magnitude performance improvement over a `Spark` engine that consumes `Thrift` data using its default deserializer. + +### Miscellanious Helpers + +Files: +- TFieldData.java : Holds the type and id members of a TField into a single int. This encoding scheme obviates the need to instantiate TField during the partial deserialization process. +- EnumCache.java : Provides a memoized way to lookup an enum by its value. +- PartialThriftComparer.java : Enables comparison of two TBase instances such that the comparison is limited to the subset of fields defined by the supplied metadata. diff --git a/lib/java/src/org/apache/thrift/partial/TFieldData.java b/lib/java/src/org/apache/thrift/partial/TFieldData.java new file mode 100644 index 000000000..9ba1a17ce --- /dev/null +++ b/lib/java/src/org/apache/thrift/partial/TFieldData.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.partial; + +/** + * Holds the type and id members of a {@link org.apache.thrift.protocol.TField} into a single int. + * + * This encoding scheme obviates the need to instantiate TField + * during the partial deserialization process. + */ +public class TFieldData { + public static int encode(byte type) { + return (int) (type & 0xff); + } + + public static int encode(byte type, short id) { + return (type & 0xff) | (((int) id) << 8); + } + + public static byte getType(int data) { + return (byte) (0xff & data); + } + + public static short getId(int data) { + return (short) ((0xffff00 & data) >> 8); + } +} diff --git a/lib/java/src/org/apache/thrift/partial/ThriftField.java b/lib/java/src/org/apache/thrift/partial/ThriftField.java new file mode 100644 index 000000000..1b5a08c80 --- /dev/null +++ b/lib/java/src/org/apache/thrift/partial/ThriftField.java @@ -0,0 +1,202 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.partial; + +import org.apache.thrift.partial.Validate; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; + +/** + * Holds name of a thrift field and of its sub-fields recursively. + * <p> + * This class is meant to be used in conjunction with {@code TDeserializer}. + */ +public class ThriftField { + + /** + * Name of this field as it appears in a thrift file. Case sensitive. + */ + public final String name; + + /** + * List of sub-fields of this field. + * + * This list should have only those sub-fields that need to be deserialized + * by the {@code TDeserializer}. + */ + public final List<ThriftField> fields; + + /** + * Constructs a {@link ThriftField}. + * + * @param name the name of this field as it appears in a thrift file. Case sensitive. + * @param fields List of sub-fields of this field. + */ + ThriftField(String name, List<ThriftField> fields) { + Validate.checkNotNullAndNotEmpty(name, "name"); + Validate.checkNotNull(fields, "fields"); + + this.name = name; + this.fields = Collections.unmodifiableList(fields); + } + + /** + * Constructs a {@link ThriftField} that does not have any sub-fields. + */ + ThriftField(String name) { + this(name, Collections.emptyList()); + } + + // Internal-only constructor that does not mark fields as read-only. + // That allows fromNames() to construct fields from names. + // The actual value of allowFieldAdds is ignored. + // It is used only for generating a different function signature. + ThriftField(String name, List<ThriftField> fields, boolean allowFieldAdds) { + Validate.checkNotNullAndNotEmpty(name, "name"); + Validate.checkNotNull(fields, "fields"); + + this.name = name; + this.fields = fields; + } + + private int hashcode = 0; + + @Override + public int hashCode() { + if (this.hashcode == 0) { + int hc = this.name.toLowerCase().hashCode(); + for (ThriftField subField : this.fields) { + hc ^= subField.hashCode(); + } + + this.hashcode = hc; + } + + return this.hashcode; + } + + @Override + public boolean equals(Object o) { + if (o == null) { + return false; + } + + if (!(o instanceof ThriftField)) { + return false; + } + + ThriftField other = (ThriftField) o; + + if (!this.name.equalsIgnoreCase(other.name)) { + return false; + } + + if (this.fields.size() != other.fields.size()) { + return false; + } + + for (int i = 0; i < this.fields.size(); i++) { + if (!this.fields.get(i).equals(other.fields.get(i))) { + return false; + } + } + + return true; + } + + @Override + public String toString() { + return String.join(", ", this.getFieldNames()); + } + + public List<String> getFieldNames() { + List<String> fieldsList = new ArrayList<>(); + if (this.fields.size() == 0) { + fieldsList.add(this.name); + } else { + for (ThriftField f : this.fields) { + for (String subF : f.getFieldNames()) { + fieldsList.add(this.name + "." + subF); + } + } + } + + return fieldsList; + } + + /** + * Generates and returns n-ary tree of fields and their sub-fields. + * <p> + * @param fieldNames collection of fully qualified field names. + * + * for example, + * In case of PinJoin thrift struct, the following are valid field names + * -- signature + * -- pins.user.userId + * -- textSignal.termSignal.termDataMap + * + * @return n-ary tree of fields and their sub-fields. + */ + public static List<ThriftField> fromNames(Collection<String> fieldNames) { + Validate.checkNotNullAndNotEmpty(fieldNames, "fieldNames"); + + List<String> fieldNamesList = new ArrayList<>(fieldNames); + Collections.sort(fieldNamesList, String.CASE_INSENSITIVE_ORDER); + + List<ThriftField> fields = new ArrayList<>(); + + for (String fieldName : fieldNamesList) { + List<ThriftField> tfields = fields; + String[] tokens = fieldName.split("\\."); + + for (String token : tokens) { + ThriftField field = findField(token, tfields); + if (field == null) { + field = new ThriftField(token, new ArrayList<>(), true); + tfields.add(field); + } + tfields = field.fields; + } + } + + return makeReadOnly(fields); + } + + private static ThriftField findField(String name, List<ThriftField> fields) { + for (ThriftField field : fields) { + if (field.name.equalsIgnoreCase(name)) { + return field; + } + } + return null; + } + + private static List<ThriftField> makeReadOnly(List<ThriftField> fields) { + List<ThriftField> result = new ArrayList<>(fields.size()); + for (ThriftField field : fields) { + ThriftField copy = new ThriftField(field.name, makeReadOnly(field.fields)); + result.add(copy); + } + return Collections.unmodifiableList(result); + } +} diff --git a/lib/java/src/org/apache/thrift/partial/ThriftFieldValueProcessor.java b/lib/java/src/org/apache/thrift/partial/ThriftFieldValueProcessor.java new file mode 100644 index 000000000..33982d1d2 --- /dev/null +++ b/lib/java/src/org/apache/thrift/partial/ThriftFieldValueProcessor.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.partial; + +import org.apache.thrift.TEnum; +import org.apache.thrift.TFieldIdEnum; + +import java.nio.ByteBuffer; + +/** + * Provides an abstraction to process deserialized field values and place them + * into the collection that holds them. This abstraction allows different types + * of collections to be output from partial deserialization. + * + * In case of the usual Thrift deserialization, the collection that holds field + * values is simply an instance of TBase. + */ +public interface ThriftFieldValueProcessor<V> { + + // Struct related methods; + Object createNewStruct(ThriftMetadata.ThriftStruct metadata); + + V prepareStruct(Object instance); + + void setBool(V valueCollection, TFieldIdEnum fieldId, boolean value); + + void setByte(V valueCollection, TFieldIdEnum fieldId, byte value); + + void setInt16(V valueCollection, TFieldIdEnum fieldId, short value); + + void setInt32(V valueCollection, TFieldIdEnum fieldId, int value); + + void setInt64(V valueCollection, TFieldIdEnum fieldId, long value); + + void setDouble(V valueCollection, TFieldIdEnum fieldId, double value); + + void setBinary(V valueCollection, TFieldIdEnum fieldId, ByteBuffer value); + + void setString(V valueCollection, TFieldIdEnum fieldId, ByteBuffer buffer); + + void setEnumField(V valueCollection, TFieldIdEnum fieldId, Object value); + + void setListField(V valueCollection, TFieldIdEnum fieldId, Object value); + + void setMapField(V valueCollection, TFieldIdEnum fieldId, Object value); + + void setSetField(V valueCollection, TFieldIdEnum fieldId, Object value); + + void setStructField(V valueCollection, TFieldIdEnum fieldId, Object value); + + Object prepareEnum(Class<? extends TEnum> enumClass, int ordinal); + + Object prepareString(ByteBuffer buffer); + + Object prepareBinary(ByteBuffer buffer); + + // List field related methods. + Object createNewList(int expectedSize); + + void setListElement(Object instance, int index, Object value); + + Object prepareList(Object instance); + + // Map field related methods. + Object createNewMap(int expectedSize); + + void setMapElement(Object instance, int index, Object key, Object value); + + Object prepareMap(Object instance); + + // Set field related methods. + Object createNewSet(int expectedSize); + + void setSetElement(Object instance, int index, Object value); + + Object prepareSet(Object instance); +} diff --git a/lib/java/src/org/apache/thrift/partial/ThriftMetadata.java b/lib/java/src/org/apache/thrift/partial/ThriftMetadata.java new file mode 100644 index 000000000..984d97249 --- /dev/null +++ b/lib/java/src/org/apache/thrift/partial/ThriftMetadata.java @@ -0,0 +1,608 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.partial; + +import org.apache.commons.lang3.StringUtils; +import org.apache.thrift.TBase; +import org.apache.thrift.TFieldIdEnum; +import org.apache.thrift.TFieldRequirementType; +import org.apache.thrift.TUnion; +import org.apache.thrift.meta_data.FieldMetaData; +import org.apache.thrift.meta_data.FieldValueMetaData; +import org.apache.thrift.meta_data.ListMetaData; +import org.apache.thrift.meta_data.MapMetaData; +import org.apache.thrift.meta_data.SetMetaData; +import org.apache.thrift.meta_data.StructMetaData; +import org.apache.thrift.partial.Validate; +import org.apache.thrift.protocol.TType; + +import java.io.Serializable; +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Container for Thrift metadata classes such as {@link ThriftPrimitive}, + * {@link ThriftList}, etc. + * <p> + * This class is mainly used by {@code TDeserializer}. + */ +public class ThriftMetadata { + + enum FieldTypeEnum implements TFieldIdEnum { + ROOT((short) 0, "root"), + ENUM((short) 1, "enum"), + LIST_ELEMENT((short) 2, "listElement"), + MAP_KEY((short) 3, "mapKey"), + MAP_VALUE((short) 4, "mapValue"), + SET_ELEMENT((short) 5, "setElement"); + + private short id; + private String name; + + FieldTypeEnum(short id, String name) { + this.id = id; + this.name = name; + } + + @Override + public short getThriftFieldId() { + return id; + } + + @Override + public String getFieldName() { + return name; + } + } + + private enum ComparisonResult { + UNKNOWN, + EQUAL, + NOT_EQUAL + } + + /** + * Base class of field types that can be partially deserialized. + * + * Holds metadata necessary for partial deserialization. + * The metadata is internally computed and used; therefore it is not visible to + * the users of {@code TDeserializer}. + */ + public abstract static class ThriftObject implements Serializable { + public final ThriftObject parent; + public final TFieldIdEnum fieldId; + public final FieldMetaData data; + + // Placeholder to attach additional data. This class or its descendents + // do not try to access or interpret this field. + public Object additionalData; + + ThriftObject(ThriftObject parent, TFieldIdEnum fieldId, FieldMetaData data) { + this.parent = parent; + this.fieldId = fieldId; + this.data = data; + } + + /** + * Converts this instance to formatted and indented string representation. + * + * @param sb the {@code StringBuilder} to add formatted strings to. + * @param level the current indent level. + */ + protected abstract void toPrettyString(StringBuilder sb, int level); + + /** + * Gets a space string whose length is proportional to the given indent level. + */ + protected String getIndent(int level) { + return StringUtils.repeat(" ", level * 4); + } + + /** + * Helper method to append a formatted string to the given {@code StringBuilder}. + */ + protected void append(StringBuilder sb, String format, Object... args) { + sb.append(String.format(format, args)); + } + + /** + * Gets the name of this field. + */ + protected String getName() { + return this.fieldId.getFieldName(); + } + + protected List<String> noFields = Collections.emptyList(); + + protected String getSubElementName(TFieldIdEnum fieldId) { + return getSubElementName(fieldId, "element"); + } + + protected String getSubElementName(TFieldIdEnum fieldId, String suffix) { + return String.format("%s_%s", fieldId.getFieldName(), suffix); + } + + private static class Factory { + + static ThriftObject createNew( + ThriftObject parent, + TFieldIdEnum fieldId, + FieldMetaData data, + List<ThriftField> fields) { + + byte fieldType = data.valueMetaData.type; + switch (fieldType) { + case TType.STRUCT: + return ThriftStructBase.create(parent, fieldId, data, fields); + + case TType.LIST: + return new ThriftList(parent, fieldId, data, fields); + + case TType.MAP: + return new ThriftMap(parent, fieldId, data, fields); + + case TType.SET: + return new ThriftSet(parent, fieldId, data, fields); + + case TType.ENUM: + return new ThriftEnum(parent, fieldId, data); + + case TType.BOOL: + case TType.BYTE: + case TType.I16: + case TType.I32: + case TType.I64: + case TType.DOUBLE: + case TType.STRING: + return new ThriftPrimitive(parent, fieldId, data); + + default: + throw unsupportedFieldTypeException(fieldType); + } + } + } + } + + /** + * Metadata about primitive types. + */ + public static class ThriftPrimitive extends ThriftObject { + ThriftPrimitive(ThriftObject parent, TFieldIdEnum fieldId, FieldMetaData data) { + super(parent, fieldId, data); + } + + public boolean isBinary() { + return this.data.valueMetaData.isBinary(); + } + + @Override + protected void toPrettyString(StringBuilder sb, int level) { + String fieldType = this.getTypeName(); + this.append(sb, "%s%s %s;\n", this.getIndent(level), fieldType, this.getName()); + } + + private String getTypeName() { + byte fieldType = this.data.valueMetaData.type; + switch (fieldType) { + case TType.BOOL: + return "bool"; + + case TType.BYTE: + return "byte"; + + case TType.I16: + return "i16"; + + case TType.I32: + return "i32"; + + case TType.I64: + return "i64"; + + case TType.DOUBLE: + return "double"; + + case TType.STRING: + if (this.isBinary()) { + return "binary"; + } else { + return "string"; + } + + default: + throw unsupportedFieldTypeException(fieldType); + } + } + + private ThriftStruct getParentStruct() { + ThriftObject tparent = parent; + while (tparent != null) { + if (tparent instanceof ThriftStruct) { + return (ThriftStruct) tparent; + } + tparent = tparent.parent; + } + return null; + } + } + + public static class ThriftEnum extends ThriftObject { + private static EnumCache enums = new EnumCache(); + + ThriftEnum(ThriftObject parent, TFieldIdEnum fieldId, FieldMetaData data) { + super(parent, fieldId, data); + } + + @Override + protected void toPrettyString(StringBuilder sb, int level) { + this.append(sb, "%senum %s;\n", this.getIndent(level), this.getName()); + } + } + + /** + * Metadata of container like objects: list, set, map + */ + public abstract static class ThriftContainer extends ThriftObject { + + public ThriftContainer( + ThriftObject parent, + TFieldIdEnum fieldId, + FieldMetaData data) { + super(parent, fieldId, data); + } + + public abstract boolean hasUnion(); + } + + public static class ThriftList extends ThriftContainer { + public final ThriftObject elementData; + + ThriftList( + ThriftObject parent, + TFieldIdEnum fieldId, + FieldMetaData data, + List<ThriftField> fields) { + super(parent, fieldId, data); + + this.elementData = ThriftObject.Factory.createNew( + this, + FieldTypeEnum.LIST_ELEMENT, + new FieldMetaData( + getSubElementName(fieldId), + TFieldRequirementType.REQUIRED, + ((ListMetaData) data.valueMetaData).elemMetaData), + fields); + } + + @Override + public boolean hasUnion() { + return this.elementData instanceof ThriftUnion; + } + + @Override + protected void toPrettyString(StringBuilder sb, int level) { + this.append(sb, "%slist<\n", this.getIndent(level)); + this.elementData.toPrettyString(sb, level + 1); + this.append(sb, "%s> %s;\n", this.getIndent(level), this.getName()); + } + } + + public static class ThriftSet extends ThriftContainer { + public final ThriftObject elementData; + + ThriftSet( + ThriftObject parent, + TFieldIdEnum fieldId, + FieldMetaData data, + List<ThriftField> fields) { + super(parent, fieldId, data); + + this.elementData = ThriftObject.Factory.createNew( + this, + FieldTypeEnum.SET_ELEMENT, + new FieldMetaData( + getSubElementName(fieldId), + TFieldRequirementType.REQUIRED, + ((SetMetaData) data.valueMetaData).elemMetaData), + fields); + } + + @Override + public boolean hasUnion() { + return this.elementData instanceof ThriftUnion; + } + + @Override + protected void toPrettyString(StringBuilder sb, int level) { + this.append(sb, "%sset<\n", this.getIndent(level)); + this.elementData.toPrettyString(sb, level + 1); + this.append(sb, "%s> %s;\n", this.getIndent(level), this.getName()); + } + } + + public static class ThriftMap extends ThriftContainer { + public final ThriftObject keyData; + public final ThriftObject valueData; + + ThriftMap( + ThriftObject parent, + TFieldIdEnum fieldId, + FieldMetaData data, + List<ThriftField> fields) { + super(parent, fieldId, data); + + this.keyData = ThriftObject.Factory.createNew( + this, + FieldTypeEnum.MAP_KEY, + new FieldMetaData( + getSubElementName(fieldId, "key"), + TFieldRequirementType.REQUIRED, + ((MapMetaData) data.valueMetaData).keyMetaData), + Collections.emptyList()); + + this.valueData = ThriftObject.Factory.createNew( + this, + FieldTypeEnum.MAP_VALUE, + new FieldMetaData( + getSubElementName(fieldId, "value"), + TFieldRequirementType.REQUIRED, + ((MapMetaData) data.valueMetaData).valueMetaData), + fields); + } + + @Override + public boolean hasUnion() { + return (this.keyData instanceof ThriftUnion) || (this.valueData instanceof ThriftUnion); + } + + @Override + protected void toPrettyString(StringBuilder sb, int level) { + this.append(sb, "%smap<\n", this.getIndent(level)); + this.append(sb, "%skey = {\n", this.getIndent(level + 1)); + this.keyData.toPrettyString(sb, level + 2); + this.append(sb, "%s},\n", this.getIndent(level + 1)); + this.append(sb, "%svalue = {\n", this.getIndent(level + 1)); + this.valueData.toPrettyString(sb, level + 2); + this.append(sb, "%s}\n", this.getIndent(level + 1)); + this.append(sb, "%s> %s;\n", this.getIndent(level), this.getName()); + } + } + + /** + * Base class for metadata of ThriftStruct and ThriftUnion. + * Holds functionality that is common to both. + */ + public abstract static class ThriftStructBase<U extends TBase> extends ThriftObject { + public ThriftStructBase( + ThriftObject parent, + TFieldIdEnum fieldId, + FieldMetaData data) { + super(parent, fieldId, data); + } + + public Class<U> getStructClass() { + return getStructClass(this.data); + } + + public static <U extends TBase> Class<U> getStructClass(FieldMetaData data) { + return (Class<U>) ((StructMetaData) data.valueMetaData).structClass; + } + + public boolean isUnion() { + return isUnion(this.data); + } + + public static boolean isUnion(FieldMetaData data) { + return TUnion.class.isAssignableFrom(getStructClass(data)); + } + + public static ThriftStructBase create( + ThriftObject parent, + TFieldIdEnum fieldId, + FieldMetaData data, + Iterable<ThriftField> fieldsData) { + + if (isUnion(data)) { + return new ThriftUnion(parent, fieldId, data, fieldsData); + } else { + return new ThriftStruct(parent, fieldId, data, fieldsData); + } + } + } + + /** + * Metadata of a Thrift union. + * Currently not adequately supported. + */ + public static class ThriftUnion<U extends TBase> extends ThriftStructBase { + public ThriftUnion( + ThriftObject parent, + TFieldIdEnum fieldId, + FieldMetaData data, + Iterable<ThriftField> fieldsData) { + super(parent, fieldId, data); + } + + @Override + protected void toPrettyString(StringBuilder sb, int level) { + String indent = this.getIndent(level); + String indent2 = this.getIndent(level + 1); + this.append(sb, "%sunion %s {\n", indent, this.getName()); + this.append(sb, "%s// unions not adequately supported at present.\n", indent2); + this.append(sb, "%s}\n", indent); + } + } + + /** + * Metadata of a Thrift struct. + */ + public static class ThriftStruct<U extends TBase> extends ThriftStructBase { + public final Map<Integer, ThriftObject> fields; + + ThriftStruct( + ThriftObject parent, + TFieldIdEnum fieldId, + FieldMetaData data, + Iterable<ThriftField> fieldsData) { + super(parent, fieldId, data); + + Class<U> clasz = getStructClass(data); + this.fields = getFields(this, clasz, fieldsData); + } + + public <T extends TBase> T createNewStruct() { + T instance = null; + + try { + Class<T> structClass = getStructClass(this.data); + instance = (T) structClass.newInstance(); + } catch (InstantiationException e) { + throw new RuntimeException(e); + } catch (IllegalAccessException e) { + throw new RuntimeException(e); + } + + return instance; + } + + public static <T extends TBase> ThriftStruct of(Class<T> clasz) { + return ThriftStruct.fromFields(clasz, Collections.emptyList()); + } + + public static <T extends TBase> ThriftStruct fromFieldNames( + Class<T> clasz, + Collection<String> fieldNames) { + return fromFields(clasz, ThriftField.fromNames(fieldNames)); + } + + public static <T extends TBase> ThriftStruct fromFields( + Class<T> clasz, + Iterable<ThriftField> fields) { + + Validate.checkNotNull(clasz, "clasz"); + Validate.checkNotNull(fields, "fields"); + + return new ThriftStruct( + null, + FieldTypeEnum.ROOT, + new FieldMetaData( + FieldTypeEnum.ROOT.getFieldName(), + TFieldRequirementType.REQUIRED, + new StructMetaData(TType.STRUCT, clasz)), + fields); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + this.toPrettyString(sb, 0); + return sb.toString(); + } + + @Override + protected void toPrettyString(StringBuilder sb, int level) { + String indent = this.getIndent(level); + String indent2 = this.getIndent(level + 1); + this.append(sb, "%sstruct %s {\n", indent, this.getName()); + if (this.fields.size() == 0) { + this.append(sb, "%s*;", indent2); + } else { + List<Integer> ids = new ArrayList(this.fields.keySet()); + Collections.sort(ids); + for (Integer id : ids) { + this.fields.get(id).toPrettyString(sb, level + 1); + } + } + this.append(sb, "%s}\n", indent); + } + + private static <U extends TBase> Map<Integer, ThriftObject> getFields( + ThriftStruct parent, + Class<U> clasz, + Iterable<ThriftField> fieldsData) { + + Map<? extends TFieldIdEnum, FieldMetaData> fieldsMetaData = + FieldMetaData.getStructMetaDataMap(clasz); + Map<Integer, ThriftObject> fields = new HashMap(); + boolean getAllFields = !fieldsData.iterator().hasNext(); + + if (getAllFields) { + for (Map.Entry<? extends TFieldIdEnum, FieldMetaData> entry : fieldsMetaData.entrySet()) { + TFieldIdEnum fieldId = entry.getKey(); + FieldMetaData fieldMetaData = entry.getValue(); + ThriftObject field = + ThriftObject.Factory.createNew(parent, fieldId, fieldMetaData, Collections.emptyList()); + fields.put((int) fieldId.getThriftFieldId(), field); + } + } else { + for (ThriftField fieldData : fieldsData) { + String fieldName = fieldData.name; + FieldMetaData fieldMetaData = findFieldMetaData(fieldsMetaData, fieldName); + TFieldIdEnum fieldId = findFieldId(fieldsMetaData, fieldName); + ThriftObject field = + ThriftObject.Factory.createNew(parent, fieldId, fieldMetaData, fieldData.fields); + fields.put((int) fieldId.getThriftFieldId(), field); + } + } + + return fields; + } + + private static FieldMetaData findFieldMetaData( + Map<? extends TFieldIdEnum, FieldMetaData> fieldsMetaData, + String fieldName) { + + for (FieldMetaData fieldData : fieldsMetaData.values()) { + if (fieldData.fieldName.equals(fieldName)) { + return fieldData; + } + } + + throw fieldNotFoundException(fieldName); + } + + private static TFieldIdEnum findFieldId( + Map<? extends TFieldIdEnum, FieldMetaData> fieldsMetaData, + String fieldName) { + + for (TFieldIdEnum fieldId : fieldsMetaData.keySet()) { + if (fieldId.getFieldName().equals(fieldName)) { + return fieldId; + } + } + + throw fieldNotFoundException(fieldName); + } + } + + static IllegalArgumentException fieldNotFoundException(String fieldName) { + return new IllegalArgumentException("field not found: '" + fieldName + "'"); + } + + static UnsupportedOperationException unsupportedFieldTypeException(byte fieldType) { + return new UnsupportedOperationException("field type not supported: '" + fieldType + "'"); + } +} diff --git a/lib/java/src/org/apache/thrift/partial/ThriftStructProcessor.java b/lib/java/src/org/apache/thrift/partial/ThriftStructProcessor.java new file mode 100644 index 000000000..95789144d --- /dev/null +++ b/lib/java/src/org/apache/thrift/partial/ThriftStructProcessor.java @@ -0,0 +1,182 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.partial; + +import org.apache.thrift.TBase; +import org.apache.thrift.TEnum; +import org.apache.thrift.TFieldIdEnum; + +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; + +/** + * Provides a way to create and initialize an instance of TBase during partial deserialization. + * + * This class is supposed to be used as a helper class for {@code PartialThriftDeserializer}. + */ +public class ThriftStructProcessor implements ThriftFieldValueProcessor<TBase> { + + private static final EnumCache enums = new EnumCache(); + + @Override + public Object createNewStruct(ThriftMetadata.ThriftStruct metadata) { + return metadata.createNewStruct(); + } + + @Override + public TBase prepareStruct(Object instance) { + return (TBase) instance; + } + + @Override + public Object createNewList(int expectedSize) { + return new Object[expectedSize]; + } + + @Override + public void setListElement(Object instance, int index, Object value) { + ((Object[]) instance)[index] = value; + } + + @Override + public Object prepareList(Object instance) { + return Arrays.asList((Object[]) instance); + } + + @Override + public Object createNewMap(int expectedSize) { + return new HashMap<Object, Object>(expectedSize); + } + + @Override + public void setMapElement(Object instance, int index, Object key, Object value) { + ((HashMap<Object, Object>) instance).put(key, value); + } + + @Override + public Object prepareMap(Object instance) { + return instance; + } + + @Override + public Object createNewSet(int expectedSize) { + return new HashSet<Object>(expectedSize); + } + + @Override + public void setSetElement(Object instance, int index, Object value) { + ((HashSet<Object>) instance).add(value); + } + + @Override + public Object prepareSet(Object instance) { + return instance; + } + + @Override + public Object prepareEnum(Class<? extends TEnum> enumClass, int ordinal) { + return enums.get(enumClass, ordinal); + } + + @Override + public Object prepareString(ByteBuffer buffer) { + return byteBufferToString(buffer); + } + + @Override + public Object prepareBinary(ByteBuffer buffer) { + return buffer; + } + + @Override + public void setBool(TBase valueCollection, TFieldIdEnum fieldId, boolean value) { + valueCollection.setFieldValue(fieldId, value); + } + + @Override + public void setByte(TBase valueCollection, TFieldIdEnum fieldId, byte value) { + valueCollection.setFieldValue(fieldId, value); + } + + @Override + public void setInt16(TBase valueCollection, TFieldIdEnum fieldId, short value) { + valueCollection.setFieldValue(fieldId, value); + } + + @Override + public void setInt32(TBase valueCollection, TFieldIdEnum fieldId, int value) { + valueCollection.setFieldValue(fieldId, value); + } + + @Override + public void setInt64(TBase valueCollection, TFieldIdEnum fieldId, long value) { + valueCollection.setFieldValue(fieldId, value); + } + + @Override + public void setDouble(TBase valueCollection, TFieldIdEnum fieldId, double value) { + valueCollection.setFieldValue(fieldId, value); + } + + @Override + public void setBinary(TBase valueCollection, TFieldIdEnum fieldId, ByteBuffer value) { + valueCollection.setFieldValue(fieldId, value); + } + + @Override + public void setString(TBase valueCollection, TFieldIdEnum fieldId, ByteBuffer buffer) { + String value = byteBufferToString(buffer); + valueCollection.setFieldValue(fieldId, value); + } + + @Override + public void setEnumField(TBase valueCollection, TFieldIdEnum fieldId, Object value) { + valueCollection.setFieldValue(fieldId, value); + } + + @Override + public void setListField(TBase valueCollection, TFieldIdEnum fieldId, Object value) { + valueCollection.setFieldValue(fieldId, value); + } + + @Override + public void setMapField(TBase valueCollection, TFieldIdEnum fieldId, Object value) { + valueCollection.setFieldValue(fieldId, value); + } + + @Override + public void setSetField(TBase valueCollection, TFieldIdEnum fieldId, Object value) { + valueCollection.setFieldValue(fieldId, value); + } + + @Override + public void setStructField(TBase valueCollection, TFieldIdEnum fieldId, Object value) { + valueCollection.setFieldValue(fieldId, value); + } + + private static String byteBufferToString(ByteBuffer buffer) { + byte[] bytes = buffer.array(); + int pos = buffer.position(); + return new String(bytes, pos, buffer.limit() - pos, StandardCharsets.UTF_8); + } +} diff --git a/lib/java/src/org/apache/thrift/partial/Validate.java b/lib/java/src/org/apache/thrift/partial/Validate.java new file mode 100644 index 000000000..ef0466a5e --- /dev/null +++ b/lib/java/src/org/apache/thrift/partial/Validate.java @@ -0,0 +1,305 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.partial; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Collection; + +/** + * A superset of Validate class in Apache commons lang3. + * + * It provides consistent message strings for frequently encountered checks. + * That simplifies callers because they have to supply only the name of the argument + * that failed a check instead of having to supply the entire message. + */ +public final class Validate { + private Validate() {} + + /** + * Validates that the given reference argument is not null. + */ + public static void checkNotNull(Object obj, String argName) { + checkArgument(obj != null, "'%s' must not be null.", argName); + } + + /** + * Validates that the given integer argument is not zero or negative. + */ + public static void checkPositiveInteger(long value, String argName) { + checkArgument(value > 0, "'%s' must be a positive integer.", argName); + } + + /** + * Validates that the given integer argument is not negative. + */ + public static void checkNotNegative(long value, String argName) { + checkArgument(value >= 0, "'%s' must not be negative.", argName); + } + + /* + * Validates that the expression (that checks a required field is present) is true. + */ + public static void checkRequired(boolean isPresent, String argName) { + checkArgument(isPresent, "'%s' is required.", argName); + } + + /** + * Validates that the expression (that checks a field is valid) is true. + */ + public static void checkValid(boolean isValid, String argName) { + checkArgument(isValid, "'%s' is invalid.", argName); + } + + /** + * Validates that the expression (that checks a field is valid) is true. + */ + public static void checkValid(boolean isValid, String argName, String validValues) { + checkArgument(isValid, "'%s' is invalid. Valid values are: %s.", argName, validValues); + } + + /** + * Validates that the given string is not null and has non-zero length. + */ + public static void checkNotNullAndNotEmpty(String arg, String argName) { + Validate.checkNotNull(arg, argName); + Validate.checkArgument( + arg.length() > 0, + "'%s' must not be empty.", + argName); + } + + /** + * Validates that the given array is not null and has at least one element. + */ + public static <T> void checkNotNullAndNotEmpty(T[] array, String argName) { + Validate.checkNotNull(array, argName); + checkNotEmpty(array.length, argName); + } + + /** + * Validates that the given array is not null and has at least one element. + */ + public static void checkNotNullAndNotEmpty(byte[] array, String argName) { + Validate.checkNotNull(array, argName); + checkNotEmpty(array.length, argName); + } + + /** + * Validates that the given array is not null and has at least one element. + */ + public static void checkNotNullAndNotEmpty(short[] array, String argName) { + Validate.checkNotNull(array, argName); + checkNotEmpty(array.length, argName); + } + + /** + * Validates that the given array is not null and has at least one element. + */ + public static void checkNotNullAndNotEmpty(int[] array, String argName) { + Validate.checkNotNull(array, argName); + checkNotEmpty(array.length, argName); + } + + /** + * Validates that the given array is not null and has at least one element. + */ + public static void checkNotNullAndNotEmpty(long[] array, String argName) { + Validate.checkNotNull(array, argName); + checkNotEmpty(array.length, argName); + } + + /** + * Validates that the given buffer is not null and has non-zero capacity. + */ + public static <T> void checkNotNullAndNotEmpty(Iterable<T> iter, String argName) { + Validate.checkNotNull(iter, argName); + int minNumElements = iter.iterator().hasNext() ? 1 : 0; + checkNotEmpty(minNumElements, argName); + } + + /** + * Validates that the given set is not null and has an exact number of items. + */ + public static <T> void checkNotNullAndNumberOfElements( + Collection<T> collection, int numElements, String argName) { + Validate.checkNotNull(collection, argName); + checkArgument( + collection.size() == numElements, + "Number of elements in '%s' must be exactly %s, %s given.", + argName, + numElements, + collection.size() + ); + } + + /** + * Validates that the given two values are equal. + */ + public static void checkValuesEqual( + long value1, + String value1Name, + long value2, + String value2Name) { + checkArgument( + value1 == value2, + "'%s' (%s) must equal '%s' (%s).", + value1Name, + value1, + value2Name, + value2); + } + + /** + * Validates that the first value is an integer multiple of the second value. + */ + public static void checkIntegerMultiple( + long value1, + String value1Name, + long value2, + String value2Name) { + checkArgument( + (value1 % value2) == 0, + "'%s' (%s) must be an integer multiple of '%s' (%s).", + value1Name, + value1, + value2Name, + value2); + } + + /** + * Validates that the first value is greater than the second value. + */ + public static void checkGreater( + long value1, + String value1Name, + long value2, + String value2Name) { + checkArgument( + value1 > value2, + "'%s' (%s) must be greater than '%s' (%s).", + value1Name, + value1, + value2Name, + value2); + } + + /** + * Validates that the first value is greater than or equal to the second value. + */ + public static void checkGreaterOrEqual( + long value1, + String value1Name, + long value2, + String value2Name) { + checkArgument( + value1 >= value2, + "'%s' (%s) must be greater than or equal to '%s' (%s).", + value1Name, + value1, + value2Name, + value2); + } + + /** + * Validates that the first value is less than or equal to the second value. + */ + public static void checkLessOrEqual( + long value1, + String value1Name, + long value2, + String value2Name) { + checkArgument( + value1 <= value2, + "'%s' (%s) must be less than or equal to '%s' (%s).", + value1Name, + value1, + value2Name, + value2); + } + + /** + * Validates that the given value is within the given range of values. + */ + public static void checkWithinRange( + long value, + String valueName, + long minValueInclusive, + long maxValueInclusive) { + checkArgument( + (value >= minValueInclusive) && (value <= maxValueInclusive), + "'%s' (%s) must be within the range [%s, %s].", + valueName, + value, + minValueInclusive, + maxValueInclusive); + } + + /** + * Validates that the given value is within the given range of values. + */ + public static void checkWithinRange( + double value, + String valueName, + double minValueInclusive, + double maxValueInclusive) { + checkArgument( + (value >= minValueInclusive) && (value <= maxValueInclusive), + "'%s' (%s) must be within the range [%s, %s].", + valueName, + value, + minValueInclusive, + maxValueInclusive); + } + + public static void checkPathExists(Path path, String argName) { + checkNotNull(path, argName); + checkArgument(Files.exists(path), "Path %s (%s) does not exist.", argName, path); + } + + public static void checkPathExistsAsDir(Path path, String argName) { + checkPathExists(path, argName); + checkArgument( + Files.isDirectory(path), + "Path %s (%s) must point to a directory.", + argName, + path); + } + + public static void checkPathExistsAsFile(Path path, String argName) { + checkPathExists(path, argName); + checkArgument(Files.isRegularFile(path), "Path %s (%s) must point to a file.", argName, path); + } + + public static void checkArgument(boolean expression, String format, Object... args) { + org.apache.commons.lang3.Validate.isTrue(expression, format, args); + } + + public static void checkState(boolean expression, String format, Object... args) { + org.apache.commons.lang3.Validate.validState(expression, format, args); + } + + private static void checkNotEmpty(int arraySize, String argName) { + Validate.checkArgument( + arraySize > 0, + "'%s' must have at least one element.", + argName); + } +} |