diff --git a/arrow/array/fixed_size_list.go b/arrow/array/fixed_size_list.go index d382ebe9..ca9210ab 100644 --- a/arrow/array/fixed_size_list.go +++ b/arrow/array/fixed_size_list.go @@ -37,6 +37,9 @@ type FixedSizeList struct { var _ ListLike = (*FixedSizeList)(nil) +func (a *FixedSizeList) Validate() error { return validateFixedSizeListArray(a) } +func (a *FixedSizeList) ValidateFull() error { return validateFixedSizeListArray(a) } + // NewFixedSizeListData returns a new List array value, from data. func NewFixedSizeListData(data arrow.ArrayData) *FixedSizeList { a := &FixedSizeList{} diff --git a/arrow/array/list.go b/arrow/array/list.go index d72887bb..0379c786 100644 --- a/arrow/array/list.go +++ b/arrow/array/list.go @@ -112,6 +112,9 @@ func (a *List) GetOneForMarshal(i int) interface{} { return json.RawMessage(v) } +func (a *List) Validate() error { return validateListArray(a, false) } +func (a *List) ValidateFull() error { return validateListArray(a, true) } + func (a *List) MarshalJSON() ([]byte, error) { var buf bytes.Buffer enc := json.NewEncoder(&buf) @@ -244,6 +247,9 @@ func (a *LargeList) GetOneForMarshal(i int) interface{} { return json.RawMessage(v) } +func (a *LargeList) Validate() error { return validateLargeListArray(a, false) } +func (a *LargeList) ValidateFull() error { return validateLargeListArray(a, true) } + func (a *LargeList) MarshalJSON() ([]byte, error) { var buf bytes.Buffer enc := json.NewEncoder(&buf) diff --git a/arrow/array/validate.go b/arrow/array/validate.go index 70b7f669..b5d493e6 100644 --- a/arrow/array/validate.go +++ b/arrow/array/validate.go @@ -20,10 +20,13 @@ import ( "fmt" "github.com/apache/arrow-go/v18/arrow" + "github.com/apache/arrow-go/v18/arrow/bitutil" + "github.com/apache/arrow-go/v18/arrow/memory" ) -// Validator is implemented by array types that can validate their internal -// consistency. See Validate and ValidateFull for top-level dispatch. +// Validator is implemented by array types that provide type-specific +// consistency checks. Validate and ValidateFull also validate generic layout +// and nested child data for every array type. type Validator interface { arrow.Array // Validate performs a basic O(1) consistency check. @@ -33,32 +36,390 @@ type Validator interface { } // Validate performs a basic O(1) consistency check on arr, returning an error -// if the array's internal buffers are inconsistent. For array types that do not -// implement Validator, nil is returned. +// if the array's internal buffers or nested child data are inconsistent. // // Use this to detect corrupted data from untrusted sources such as Arrow Flight // or Flight SQL servers before accessing values, which may otherwise panic. func Validate(arr arrow.Array) error { - if v, ok := arr.(Validator); ok { - return v.Validate() - } - return nil + return validateArray(arr, false, "") } // ValidateFull performs a thorough O(n) consistency check on arr, returning an -// error if the array's internal buffers are inconsistent. For array types that -// do not implement Validator, nil is returned. +// error if the array's internal buffers or nested child data are inconsistent. // // Unlike Validate, this checks every element and is therefore O(n). Use this // when receiving data from untrusted sources where subtle corruption (e.g. // non-monotonic offsets) may not be detected by Validate alone. func ValidateFull(arr arrow.Array) error { + return validateArray(arr, true, "") +} + +func validateArray(arr arrow.Array, full bool, path string) error { + if arr == nil { + return nil + } + + data, ok := arr.Data().(*Data) + if !ok || data == nil { + return validationError(path, fmt.Errorf("arrow/array: array does not expose internal data")) + } + + if err := validateArrayData(data); err != nil { + return validationError(path, err) + } + if err := validateArrayStructure(data); err != nil { + return validationError(path, err) + } + if v, ok := arr.(Validator); ok { - return v.ValidateFull() + var err error + if full { + err = v.ValidateFull() + } else { + err = v.Validate() + } + if err != nil { + return validationError(path, err) + } + } + + if data.DataType().Layout().HasDict { + dictData := data.dictionary + if dictData != nil { + dt := data.DataType().(*arrow.DictionaryType) + indexData := NewData(dt.IndexType, data.length, data.buffers, nil, data.nulls, data.offset) + err := checkIndexBounds(indexData, uint64(dictData.Len())) + indexData.Release() + if err != nil { + return validationError(joinValidationPath(path, "dictionary indices"), err) + } + } } + + if ext, ok := arr.(ExtensionArray); ok { + return validateArray(ext.Storage(), full, joinValidationPath(path, "storage")) + } + + for i, childData := range data.Children() { + child, err := makeArrayFromData(childData) + if err != nil { + return validationError(joinValidationPath(path, validationChildPath(data.DataType(), i)), err) + } + + childPath := joinValidationPath(path, validationChildPath(data.DataType(), i)) + err = validateArray(child, full, childPath) + child.Release() + if err != nil { + return err + } + } + + if dictData := data.dictionary; dictData != nil { + dict, err := makeArrayFromData(dictData) + if err != nil { + return validationError(joinValidationPath(path, "dictionary"), err) + } + + err = validateArray(dict, full, joinValidationPath(path, "dictionary")) + dict.Release() + if err != nil { + return err + } + } + return nil } +func validateArrayData(data *Data) error { + if data == nil || data.dtype == nil { + return fmt.Errorf("arrow/array: array data has no data type") + } + if data.offset < 0 { + return fmt.Errorf("arrow/array: array offset is negative: %d", data.offset) + } + if data.length < 0 { + return fmt.Errorf("arrow/array: array length is negative: %d", data.length) + } + if data.nulls < UnknownNullCount || data.nulls > data.length { + return fmt.Errorf("arrow/array: invalid null count %d for length %d", data.nulls, data.length) + } + + end := int64(data.offset) + int64(data.length) + if end < int64(data.offset) { + return fmt.Errorf("arrow/array: array offset and length overflow") + } + + layout := data.dtype.Layout() + if len(data.buffers) < len(layout.Buffers) { + return fmt.Errorf("arrow/array: expected at least %d buffers for %s, got %d", + len(layout.Buffers), data.dtype, len(data.buffers)) + } + if layout.VariadicSpec == nil && len(data.buffers) > len(layout.Buffers) { + return fmt.Errorf("arrow/array: expected at most %d buffers for %s, got %d", + len(layout.Buffers), data.dtype, len(data.buffers)) + } + + for i, spec := range layout.Buffers { + if err := validateBuffer(data.buffers[i], spec, end, data.length, i); err != nil { + return err + } + } + if layout.VariadicSpec != nil { + for i := len(layout.Buffers); i < len(data.buffers); i++ { + if err := validateBuffer(data.buffers[i], *layout.VariadicSpec, end, data.length, i); err != nil { + return err + } + } + } + return nil +} + +func validateBuffer(buf *memory.Buffer, spec arrow.BufferSpec, end int64, length, index int) error { + if buf == nil { + if spec.Kind == arrow.KindFixedWidth || spec.Kind == arrow.KindVarWidth { + if length > 0 { + return fmt.Errorf("arrow/array: buffer %d is nil for non-empty layout kind %v", index, spec.Kind) + } + } + return nil + } + + bufferLen := int64(buf.Len()) + switch spec.Kind { + case arrow.KindFixedWidth: + if spec.ByteWidth <= 0 { + return fmt.Errorf("arrow/array: buffer %d has invalid fixed width %d", index, spec.ByteWidth) + } + if end > bufferLen/int64(spec.ByteWidth) { + return fmt.Errorf("arrow/array: buffer %d is too small for offset %d and length %d", index, end-int64(length), length) + } + case arrow.KindBitmap: + if int64(bitutil.BytesForBits(end)) > bufferLen { + return fmt.Errorf("arrow/array: bitmap buffer %d is too small for offset %d and length %d", index, end-int64(length), length) + } + case arrow.KindVarWidth: + return nil + case arrow.KindAlwaysNull: + return fmt.Errorf("arrow/array: buffer %d must be nil for an always-null layout", index) + } + return nil +} + +func validateArrayStructure(data *Data) error { + children := data.Children() + if nested, ok := data.DataType().(arrow.NestedType); ok { + fields := nested.Fields() + if len(children) != len(fields) { + return fmt.Errorf("arrow/array: %s expects %d child arrays, got %d", data.DataType(), len(fields), len(children)) + } + for i, child := range children { + if child == nil { + return fmt.Errorf("arrow/array: child %d of %s is nil", i, data.DataType()) + } + if !arrow.TypeEqual(fields[i].Type, child.DataType()) { + return fmt.Errorf("arrow/array: child %d of %s has type %s, want %s", + i, data.DataType(), child.DataType(), fields[i].Type) + } + } + } else if len(children) != 0 { + return fmt.Errorf("arrow/array: %s must not have child arrays", data.DataType()) + } + + if data.DataType().Layout().HasDict { + dt, ok := data.DataType().(*arrow.DictionaryType) + if !ok { + return fmt.Errorf("arrow/array: datatype %s declares dictionary storage but is not a dictionary type", data.DataType()) + } + if dict := data.dictionary; dict != nil && !arrow.TypeEqual(dt.ValueType, dict.DataType()) { + return fmt.Errorf("arrow/array: dictionary has type %s, want %s", dict.DataType(), dt.ValueType) + } + if data.Len() > 0 && data.dictionary == nil { + return fmt.Errorf("arrow/array: non-empty dictionary array has no dictionary") + } + } + return nil +} + +func validateListArray(a *List, full bool) error { + if err := validateArrayData(a.data); err != nil { + return err + } + if len(a.data.childData) != 1 || a.data.childData[0] == nil { + return fmt.Errorf("arrow/array: list array must have one non-nil child array") + } + dt, ok := a.data.dtype.(arrow.ListLikeType) + if !ok || dt.ID() == arrow.LARGE_LIST || dt.ID() == arrow.LARGE_LIST_VIEW { + return fmt.Errorf("arrow/array: invalid datatype %s for list array", a.data.dtype) + } + if !arrow.TypeEqual(dt.Elem(), a.data.childData[0].DataType()) { + return fmt.Errorf("arrow/array: list values have type %s, want %s", a.data.childData[0].DataType(), dt.Elem()) + } + if len(a.offsets) == 0 && a.data.length > 0 { + return fmt.Errorf("arrow/array: non-empty list array has no offsets") + } + return validateListOffsets(a.offsets, a.data, a.data.childData[0].Len(), full) +} + +func validateLargeListArray(a *LargeList, full bool) error { + if err := validateArrayData(a.data); err != nil { + return err + } + if len(a.data.childData) != 1 || a.data.childData[0] == nil { + return fmt.Errorf("arrow/array: large list array must have one non-nil child array") + } + dt, ok := a.data.dtype.(arrow.ListLikeType) + if !ok || dt.ID() != arrow.LARGE_LIST { + return fmt.Errorf("arrow/array: invalid datatype %s for large list array", a.data.dtype) + } + if !arrow.TypeEqual(dt.Elem(), a.data.childData[0].DataType()) { + return fmt.Errorf("arrow/array: large list values have type %s, want %s", a.data.childData[0].DataType(), dt.Elem()) + } + if len(a.offsets) == 0 && a.data.length > 0 { + return fmt.Errorf("arrow/array: non-empty large list array has no offsets") + } + return validateListOffsets(a.offsets, a.data, a.data.childData[0].Len(), full) +} + +func validateListOffsets(offsets interface{}, data *Data, valueLength int, full bool) error { + if data.length == 0 { + return nil + } + start := int64(data.offset) + required := start + int64(data.length) + 1 + if required < start || required > int64(offsetLength(offsets)) { + return fmt.Errorf("arrow/array: list offsets buffer is too small for offset %d and length %d", data.offset, data.length) + } + get := func(i int64) int64 { + switch v := offsets.(type) { + case []int32: + return int64(v[int(i)]) + case []int64: + return v[int(i)] + default: + return 0 + } + } + + first := get(start) + last := get(required - 1) + if err := validateListOffset(first, start, valueLength); err != nil { + return err + } + if err := validateListOffset(last, required-1, valueLength); err != nil { + return err + } + if !full { + return nil + } + + previous := first + for i := start + 1; i < required; i++ { + current := get(i) + if current < 0 || current > int64(valueLength) { + return fmt.Errorf("arrow/array: list offset at index %d out of bounds: %d", i, current) + } + if current < previous { + return fmt.Errorf("arrow/array: list offsets are not monotonically non-decreasing at index %d: %d < %d", + i, current, previous) + } + previous = current + } + return nil +} + +func offsetLength(offsets interface{}) int { + switch v := offsets.(type) { + case []int32: + return len(v) + case []int64: + return len(v) + default: + return 0 + } +} + +func validateListOffset(offset int64, index int64, valueLength int) error { + if offset < 0 { + return fmt.Errorf("arrow/array: list offset at index %d is negative: %d", index, offset) + } + if offset > int64(valueLength) { + return fmt.Errorf("arrow/array: list offset at index %d out of bounds: %d > %d", index, offset, valueLength) + } + return nil +} + +func validateFixedSizeListArray(a *FixedSizeList) error { + if err := validateArrayData(a.data); err != nil { + return err + } + if len(a.data.childData) != 1 || a.data.childData[0] == nil { + return fmt.Errorf("arrow/array: fixed-size list array must have one non-nil child array") + } + dt, ok := a.data.dtype.(*arrow.FixedSizeListType) + if !ok { + return fmt.Errorf("arrow/array: invalid datatype %s for fixed-size list array", a.data.dtype) + } + if !arrow.TypeEqual(dt.Elem(), a.data.childData[0].DataType()) { + return fmt.Errorf("arrow/array: fixed-size list values have type %s, want %s", a.data.childData[0].DataType(), dt.Elem()) + } + childLength := int64(a.data.offset) + int64(a.data.length) + itemCount := int64(dt.Len()) + if itemCount <= 0 { + return fmt.Errorf("arrow/array: fixed-size list has invalid item count %d", itemCount) + } + if childLength > int64(a.data.childData[0].Len())/itemCount { + return fmt.Errorf("arrow/array: fixed-size list child length %d is too small for offset %d and length %d", + a.data.childData[0].Len(), a.data.offset, a.data.length) + } + return nil +} + +func makeArrayFromData(data arrow.ArrayData) (arr arrow.Array, err error) { + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("arrow/array: failed to construct child array: %v", r) + } + }() + return MakeFromData(data), nil +} + +func validationChildPath(dt arrow.DataType, index int) string { + if nested, ok := dt.(arrow.NestedType); ok { + fields := nested.Fields() + if index >= 0 && index < len(fields) { + field := fields[index] + if dt.ID() == arrow.STRUCT || dt.ID() == arrow.SPARSE_UNION || dt.ID() == arrow.DENSE_UNION || dt.ID() == arrow.RUN_END_ENCODED { + return fmt.Sprintf("field %q", field.Name) + } + } + } + + switch dt.ID() { + case arrow.LIST, arrow.LARGE_LIST, arrow.FIXED_SIZE_LIST: + return "list values" + case arrow.LIST_VIEW, arrow.LARGE_LIST_VIEW: + return "list view values" + case arrow.MAP: + return "map entries" + default: + return fmt.Sprintf("child %d", index) + } +} + +func joinValidationPath(path, child string) string { + if path == "" { + return child + } + return path + " -> " + child +} + +func validationError(path string, err error) error { + if err == nil || path == "" { + return err + } + return fmt.Errorf("%s: %w", path, err) +} + // ValidateRecord validates each column in rec using Validate, returning the // first error encountered. The error includes the column index and field name. func ValidateRecord(rec arrow.RecordBatch) error { diff --git a/arrow/array/validate_test.go b/arrow/array/validate_test.go index 5d45ce50..a3cc8a2e 100644 --- a/arrow/array/validate_test.go +++ b/arrow/array/validate_test.go @@ -17,6 +17,7 @@ package array import ( + "strings" "testing" "github.com/apache/arrow-go/v18/arrow" @@ -209,3 +210,68 @@ func TestTopLevelValidate(t *testing.T) { assert.Contains(t, err.Error(), "column 1 (bad)") }) } + +func TestTopLevelValidateFullRecursesThroughNestedChildren(t *testing.T) { + badString := makeStringArrayRaw(t, []int32{0, 5, 3, 5}, "hello", 3, 0) + defer badString.Release() + + inner, err := NewStructArrayWithFields([]arrow.Array{badString}, []arrow.Field{ + {Name: "name", Type: arrow.BinaryTypes.String, Nullable: true}, + }) + require.NoError(t, err) + defer inner.Release() + + offsets := memory.NewBufferBytes(arrow.Int32Traits.CastToBytes([]int32{0, 3})) + data := NewData(arrow.ListOf(inner.DataType()), 1, + []*memory.Buffer{nil, offsets}, []arrow.ArrayData{inner.Data()}, 0, 0) + offsets.Release() + list := NewListData(data) + data.Release() + defer list.Release() + + outer, err := NewStructArrayWithFields([]arrow.Array{list}, []arrow.Field{ + {Name: "orders", Type: list.DataType(), Nullable: true}, + }) + require.NoError(t, err) + defer outer.Release() + + err = ValidateFull(outer) + require.Error(t, err) + assert.Contains(t, err.Error(), `field "orders" -> list values -> field "name"`) +} + +func TestListValidateFullChecksOffsets(t *testing.T) { + values, _, err := FromJSON(memory.DefaultAllocator, arrow.PrimitiveTypes.Int32, strings.NewReader(`[1, 2]`)) + require.NoError(t, err) + defer values.Release() + + offsets := memory.NewBufferBytes(arrow.Int32Traits.CastToBytes([]int32{0, 2, 1})) + data := NewData(arrow.ListOf(arrow.PrimitiveTypes.Int32), 2, + []*memory.Buffer{nil, offsets}, []arrow.ArrayData{values.Data()}, 0, 0) + offsets.Release() + list := NewListData(data) + data.Release() + defer list.Release() + + assert.NoError(t, list.Validate()) + err = list.ValidateFull() + require.Error(t, err) + assert.Contains(t, err.Error(), "not monotonically non-decreasing") +} + +func TestTopLevelValidateFullRecursesDictionaryValues(t *testing.T) { + badDictionary := makeStringArrayRaw(t, []int32{0, 5, 3, 5}, "hello", 3, 0) + defer badDictionary.Release() + indices, _, err := FromJSON(memory.DefaultAllocator, arrow.PrimitiveTypes.Int8, strings.NewReader(`[0]`)) + require.NoError(t, err) + defer indices.Release() + + dt := &arrow.DictionaryType{IndexType: arrow.PrimitiveTypes.Int8, ValueType: arrow.BinaryTypes.String} + dict := NewDictionaryArray(dt, indices, badDictionary) + defer dict.Release() + + err = ValidateFull(dict) + require.Error(t, err) + assert.Contains(t, err.Error(), "dictionary") + assert.Contains(t, err.Error(), "not monotonically non-decreasing") +}