diff --git a/src/arrow2/src/io/parquet/read/deserialize/mod.rs b/src/arrow2/src/io/parquet/read/deserialize/mod.rs index c05978d74..3d641e452 100644 --- a/src/arrow2/src/io/parquet/read/deserialize/mod.rs +++ b/src/arrow2/src/io/parquet/read/deserialize/mod.rs @@ -154,7 +154,7 @@ where } nested::columns_to_iter_recursive( - columns, types, field, init, num_rows, chunk_size, num_values, + columns, types, field, init, num_rows, chunk_size, num_values, false, ) } @@ -242,7 +242,7 @@ where { Ok(Box::new( nested::columns_to_iter_recursive( - columns, types, field, init, num_rows, chunk_size, num_values, + columns, types, field, init, num_rows, chunk_size, num_values, false, )? .map(|x| x.map(|x| x.1)), )) diff --git a/src/arrow2/src/io/parquet/read/deserialize/nested.rs b/src/arrow2/src/io/parquet/read/deserialize/nested.rs index ba7125b76..5b168f25c 100644 --- a/src/arrow2/src/io/parquet/read/deserialize/nested.rs +++ b/src/arrow2/src/io/parquet/read/deserialize/nested.rs @@ -42,6 +42,7 @@ pub fn columns_to_iter_recursive<'a, I>( num_rows: usize, chunk_size: Option, mut num_values: Vec, + is_parent_nullable: bool, ) -> Result> where I: Pages + 'a, @@ -49,10 +50,11 @@ where use crate::datatypes::PhysicalType::*; use crate::datatypes::PrimitiveType::*; + let is_nullable = is_parent_nullable || field.is_nullable; Ok(match field.data_type().to_physical_type() { Null => { // physical type is i32 - init.push(InitNested::Primitive(field.is_nullable)); + init.push(InitNested::Primitive(is_nullable)); types.pop(); primitive(null::NestedIter::new( columns.pop().unwrap(), @@ -64,7 +66,7 @@ where )) } Boolean => { - init.push(InitNested::Primitive(field.is_nullable)); + init.push(InitNested::Primitive(is_nullable)); types.pop(); primitive(boolean::NestedIter::new( columns.pop().unwrap(), @@ -75,7 +77,7 @@ where )) } Primitive(Int8) => { - init.push(InitNested::Primitive(field.is_nullable)); + init.push(InitNested::Primitive(is_nullable)); types.pop(); primitive(primitive::NestedIter::new( columns.pop().unwrap(), @@ -88,7 +90,7 @@ where )) } Primitive(Int16) => { - init.push(InitNested::Primitive(field.is_nullable)); + init.push(InitNested::Primitive(is_nullable)); types.pop(); primitive(primitive::NestedIter::new( columns.pop().unwrap(), @@ -101,7 +103,7 @@ where )) } Primitive(Int32) => { - init.push(InitNested::Primitive(field.is_nullable)); + init.push(InitNested::Primitive(is_nullable)); types.pop(); primitive(primitive::NestedIter::new( columns.pop().unwrap(), @@ -114,7 +116,7 @@ where )) } Primitive(Int64) => { - init.push(InitNested::Primitive(field.is_nullable)); + init.push(InitNested::Primitive(is_nullable)); types.pop(); primitive(primitive::NestedIter::new( columns.pop().unwrap(), @@ -127,7 +129,7 @@ where )) } Primitive(UInt8) => { - init.push(InitNested::Primitive(field.is_nullable)); + init.push(InitNested::Primitive(is_nullable)); types.pop(); primitive(primitive::NestedIter::new( columns.pop().unwrap(), @@ -140,7 +142,7 @@ where )) } Primitive(UInt16) => { - init.push(InitNested::Primitive(field.is_nullable)); + init.push(InitNested::Primitive(is_nullable)); types.pop(); primitive(primitive::NestedIter::new( columns.pop().unwrap(), @@ -153,7 +155,7 @@ where )) } Primitive(UInt32) => { - init.push(InitNested::Primitive(field.is_nullable)); + init.push(InitNested::Primitive(is_nullable)); let type_ = types.pop().unwrap(); match type_.physical_type { PhysicalType::Int32 => primitive(primitive::NestedIter::new( @@ -183,7 +185,7 @@ where } } Primitive(UInt64) => { - init.push(InitNested::Primitive(field.is_nullable)); + init.push(InitNested::Primitive(is_nullable)); types.pop(); primitive(primitive::NestedIter::new( columns.pop().unwrap(), @@ -196,7 +198,7 @@ where )) } Primitive(Float32) => { - init.push(InitNested::Primitive(field.is_nullable)); + init.push(InitNested::Primitive(is_nullable)); types.pop(); primitive(primitive::NestedIter::new( columns.pop().unwrap(), @@ -209,7 +211,7 @@ where )) } Primitive(Float64) => { - init.push(InitNested::Primitive(field.is_nullable)); + init.push(InitNested::Primitive(is_nullable)); types.pop(); primitive(primitive::NestedIter::new( columns.pop().unwrap(), @@ -222,7 +224,7 @@ where )) } Binary | Utf8 => { - init.push(InitNested::Primitive(field.is_nullable)); + init.push(InitNested::Primitive(is_nullable)); types.pop(); remove_nested(binary::NestedIter::::new( columns.pop().unwrap(), @@ -234,7 +236,7 @@ where )) } LargeBinary | LargeUtf8 => { - init.push(InitNested::Primitive(field.is_nullable)); + init.push(InitNested::Primitive(is_nullable)); types.pop(); remove_nested(binary::NestedIter::::new( columns.pop().unwrap(), @@ -254,7 +256,7 @@ where // Hence, unlike the other type readers, for now we do not pass on `num_values` as // there is currently no way to encounter or test this code. This should be fixed // when support for reading the dictionary types is added. - init.push(InitNested::Primitive(field.is_nullable)); + init.push(InitNested::Primitive(is_nullable)); let type_ = types.pop().unwrap(); let iter = columns.pop().unwrap(); let data_type = field.data_type().clone(); @@ -265,7 +267,7 @@ where DataType::List(inner) | DataType::LargeList(inner) | DataType::FixedSizeList(inner, _) => { - init.push(InitNested::List(field.is_nullable)); + init.push(InitNested::List(is_nullable)); let iter = columns_to_iter_recursive( columns, types, @@ -274,6 +276,7 @@ where num_rows, chunk_size, num_values, + is_nullable, )?; let iter = iter.map(move |x| { let (mut nested, array) = x?; @@ -283,7 +286,7 @@ where Box::new(iter) as _ } DataType::Decimal(_, _) => { - init.push(InitNested::Primitive(field.is_nullable)); + init.push(InitNested::Primitive(is_nullable)); let type_ = types.pop().unwrap(); match type_.physical_type { PhysicalType::Int32 => primitive(primitive::NestedIter::new( @@ -349,7 +352,7 @@ where } } DataType::Decimal256(_, _) => { - init.push(InitNested::Primitive(field.is_nullable)); + init.push(InitNested::Primitive(is_nullable)); let type_ = types.pop().unwrap(); match type_.physical_type { PhysicalType::Int32 => primitive(primitive::NestedIter::new( @@ -452,7 +455,7 @@ where .rev() .map(|f| { let mut init = init.clone(); - init.push(InitNested::Struct(field.is_nullable)); + init.push(InitNested::Struct(is_nullable)); let n = n_columns(&f.data_type); let columns = columns.drain(columns.len() - n..).collect(); let types = types.drain(types.len() - n..).collect(); @@ -465,6 +468,7 @@ where num_rows, chunk_size, num_values, + is_nullable, ) }) .collect::>>()?; @@ -472,7 +476,7 @@ where Box::new(struct_::StructIterator::new(columns, fields.clone())) } DataType::Map(inner, _) => { - init.push(InitNested::List(field.is_nullable)); + init.push(InitNested::List(is_nullable)); let iter = columns_to_iter_recursive( columns, types, @@ -481,6 +485,7 @@ where num_rows, chunk_size, num_values, + is_nullable, )?; let iter = iter.map(move |x| { let (mut nested, array) = x?;