Add extra bytes support

[gadomski]

las 1.4 explicitly supports extra bytes after each point, and you can do it for earlier versions as well. We should support this.

[bmmeijers]

I've taken a step at modelling the extra bytes for reading. This seems to work with the one file I have that has extra bytes. Maybe this could also be part of the las library, but then an API for dealing with the extra byte vlr's needs to be defined. And, writing them is something different, I guess.

Anyway, below is my attempt:

use las::{Read, Reader};

#[derive(Debug)]
struct RawExtraBytes {
    _reserved: [u8; 2],
    data_type: [u8; 1],
    options: [u8; 1],
    name: [u8; 32],
    _unused: [u8; 4],
    // NYI: no data / min / max
    _no_data: [u8; 24],
    _min: [u8; 24],
    _max: [u8; 24],
    scale: [u8; 24],
    offset: [u8; 24],
    description: [u8; 32],
}

#[derive(Debug)]
struct RawExtraBytesOptions {
    options: u8,
}

impl RawExtraBytesOptions {
    // 0 - no_data_bit - If set the no_data value is relevant
    fn no_data_is_relevant(&self) -> bool {
        (self.options & 1) != 0
    }

    // 1 - min_bit - If set the min value is relevant
    fn min_value_is_relevant(&self) -> bool {
        (self.options & 2) != 0
    }

    // 2 - max bit - If set the max value is relevant
    fn max_value_is_relevant(&self) -> bool {
        (self.options & 4) != 0
    }

    // 3 - scale_bit - If set each value should be multiplied by the corresponding scale value (before applying the offset)
    fn scale_bit_is_set(&self) -> bool {
        (self.options & 8) != 0
    }

    // 4 - offset_bit - If set each value should be translated by the corresponding offset value (after applying the scaling).
    fn offset_bit_is_set(&self) -> bool {
        (self.options & 8) != 0
    }
}

impl RawExtraBytes {
    fn name(&self) -> &str {
        std::str::from_utf8(&self.name).unwrap()
    }

    fn options(&self) -> RawExtraBytesOptions {
        RawExtraBytesOptions {
            options: self.options[0],
        }
    }

    fn data_type(&self) -> ExtraByteDataType {
        self.data_type[0].into()
    }

    fn description(&self) -> &str {
        std::str::from_utf8(&self.description).unwrap()
    }
}

// 0 undocumented extra bytes specify value in options field
// 1 unsigned char      1 byte
// 2 char               1 byte
// 3 unsigned short     2 bytes
// 4 short              2 bytes
// 5 unsigned long      4 bytes
// 6 long               4 bytes
// 7 unsigned long long 8 bytes
// 8 long long          8 bytes
// 9 float              4 bytes
// 10 double            8 bytes
//
// 11-30 Deprecated deprecated
// 31-255 Reserved not assigned
#[derive(Debug)]
enum ExtraByteDataType {
    Undocumented = 0,
    Uchar,
    Char,
    UShort,
    Short,
    ULong,
    Long,
    ULongLong,
    LongLong,
    Float,
    Double,
    Deprecated = 11,
    Reserved = 31,
}

// Note, it seems that LasPy checks if the number is between 0 and 10 -> 1D, 20 -> 2D, 30 -> 3D
// and deal with scales / offsets in that way
// https://github.com/laspy/laspy/blob/master/laspy/vlrs/known.py#L279
impl Into<ExtraByteDataType> for u8 {
    fn into(self) -> ExtraByteDataType {
        match self {
            0 => ExtraByteDataType::Undocumented,
            1 => ExtraByteDataType::Uchar,
            2 => ExtraByteDataType::Char,
            3 => ExtraByteDataType::UShort,
            4 => ExtraByteDataType::Short,
            5 => ExtraByteDataType::ULong,
            6 => ExtraByteDataType::Long,
            7 => ExtraByteDataType::ULongLong,
            8 => ExtraByteDataType::LongLong,
            9 => ExtraByteDataType::Float,
            10 => ExtraByteDataType::Double,
            11..=30 => ExtraByteDataType::Deprecated,
            31..=255 => ExtraByteDataType::Reserved,
        }
    }
}

impl ExtraByteDataType {
    fn byte_size(&self) -> u8 {
        match self {
            ExtraByteDataType::Undocumented => 0,
            ExtraByteDataType::Uchar => 1,
            ExtraByteDataType::Char => 1,
            ExtraByteDataType::UShort => 2,
            ExtraByteDataType::Short => 2,
            ExtraByteDataType::ULong => 4,
            ExtraByteDataType::Long => 4,
            ExtraByteDataType::ULongLong => 8,
            ExtraByteDataType::LongLong => 8,
            ExtraByteDataType::Float => 4,
            ExtraByteDataType::Double => 8,
            ExtraByteDataType::Deprecated => 0,
            ExtraByteDataType::Reserved => 0,
        }
    }
}

fn main() {
    let path = std::env::args()
        .skip(1)
        .next()
        .expect("Must provide a path to a las file");
    let reader = Reader::from_path(&path).expect("Unable to open reader");
    let header = reader.header();

    // Read the ExtraBytes VLRs and store the found bytes in RawExtraBytes structs
    let mut extra_bytes = vec![];
    for vlr in header.all_vlrs() {
        if vlr.record_id == 4 {
            println!("*vlr -> {:?}", vlr.data);

            // There should be a multiple of 192 bytes
            let nr = vlr.data.len() / 192;

            for i in 0..nr {
                println!();
                // FIXME: NYI: Use Cursor on vlr.data
                let bytes = RawExtraBytes {
                    _reserved: vlr.data[0 + i * 192..2 + i * 192].try_into().unwrap(),
                    data_type: vlr.data[2 + i * 192..3 + i * 192].try_into().unwrap(),
                    options: vlr.data[3 + i * 192..4 + i * 192].try_into().unwrap(),
                    name: vlr.data[4 + i * 192..36 + i * 192].try_into().unwrap(),
                    _unused: vlr.data[36 + i * 192..40 + i * 192].try_into().unwrap(),
                    _no_data: vlr.data[40 + i * 192..64 + i * 192].try_into().unwrap(),
                    _min: vlr.data[64 + i * 192..88 + i * 192].try_into().unwrap(),
                    _max: vlr.data[88 + i * 192..112 + i * 192].try_into().unwrap(),
                    scale: vlr.data[112 + i * 192..136 + i * 192].try_into().unwrap(),
                    offset: vlr.data[136 + i * 192..160 + i * 192].try_into().unwrap(),
                    description: vlr.data[160 + i * 192..192 + i * 192].try_into().unwrap(),
                };

                println!("Extra attrib -- name: {}", bytes.name());
                println!("Extra attrib -- data_type: {:?}", bytes.data_type());
                println!("Extra attrib -- option bit: {:?}", bytes.options());

                let options = bytes.options();
                let bools = vec![
                    options.no_data_is_relevant(),
                    options.min_value_is_relevant(),
                    options.max_value_is_relevant(),
                    options.scale_bit_is_set(),
                    options.offset_bit_is_set(),
                ];
                println!("{:?}", bools);

                println!("Extra attrib its description: {}", bytes.description());

                extra_bytes.push(bytes);
            }
        }
    }

    // Now while reading the points, we can get to the values
    let mut reader = Reader::from_path(&path).expect("Unable to open reader");
    reader.points().for_each(|p| {
        let pt = p.expect("Unable to read point");
        let mut acc = 0;
        for e in &extra_bytes {
            let byte_size = e.data_type().byte_size();
            let relevant_bytes = &pt.extra_bytes[acc as usize..(acc + byte_size) as usize];
            println!();
            println!(
                "* {} -- data type [{:?}] byte size [{}], data: {:?}",
                e.name(),
                e.data_type(),
                byte_size,
                relevant_bytes
            );

            let options = e.options();
            let bools = vec![
                options.no_data_is_relevant(),
                options.min_value_is_relevant(),
                options.max_value_is_relevant(),
                options.scale_bit_is_set(),
                options.offset_bit_is_set(),
            ];
            println!("{:?}", bools);

            let mut scale = 1.0;
            if options.scale_bit_is_set() {
                println!("scale, bytes: {:?}", e.scale);
                scale = f64::from_le_bytes(e.scale[0..8].try_into().unwrap());
                for i in 0..3 {
                    let j = i + 1;
                    let tmp = f64::from_le_bytes(e.scale[i * 8..j * 8].try_into().unwrap());
                    println!("scale[{i}] = {tmp}");
                }
            }

            let mut offset = 0.0;
            if options.offset_bit_is_set() {
                println!("offset, bytes: {:?}", e.offset);
                offset = f64::from_le_bytes(e.offset[0..8].try_into().unwrap());
                for i in 0..3 {
                    let j = i + 1;
                    let tmp = f64::from_le_bytes(e.offset[i * 8..j * 8].try_into().unwrap());
                    println!("offset[{i}] = {tmp}");
                }
            }

            println!("{scale} {offset}");

            let value = match e.data_type() {
                ExtraByteDataType::Undocumented => 0.0,
                ExtraByteDataType::Uchar => {
                    u8::from_le_bytes(relevant_bytes.try_into().unwrap()) as f64
                }
                ExtraByteDataType::Char => {
                    i8::from_le_bytes(relevant_bytes.try_into().unwrap()) as f64
                }
                ExtraByteDataType::UShort => {
                    u16::from_le_bytes(relevant_bytes.try_into().unwrap()) as f64
                }
                ExtraByteDataType::Short => {
                    i16::from_le_bytes(relevant_bytes.try_into().unwrap()) as f64
                }
                ExtraByteDataType::ULong => {
                    u32::from_le_bytes(relevant_bytes.try_into().unwrap()) as f64
                }
                ExtraByteDataType::Long => {
                    i32::from_le_bytes(relevant_bytes.try_into().unwrap()) as f64
                }
                ExtraByteDataType::ULongLong => {
                    u64::from_le_bytes(relevant_bytes.try_into().unwrap()) as f64
                }
                ExtraByteDataType::LongLong => {
                    i64::from_le_bytes(relevant_bytes.try_into().unwrap()) as f64
                }
                ExtraByteDataType::Float => {
                    f32::from_le_bytes(relevant_bytes.try_into().unwrap()) as f64
                }
                ExtraByteDataType::Double => {
                    f64::from_le_bytes(relevant_bytes.try_into().unwrap()) as f64
                }
                ExtraByteDataType::Deprecated => 0.0,
                ExtraByteDataType::Reserved => 0.0,
            };

            println!(
                "{}, with scale {} and offset {} applied => {}",
                value,
                scale,
                offset,
                value * scale + offset
            );

            acc += byte_size;
        }
    });
}

[gadomski]

If you'd like to open a PR with an implementation, I'd be happy to take a look. Thanks!

[bmmeijers]

I'd first love some feedback on the code above, and then some pointers where / how I could add these VLR's into the library.

[gadomski]

I find it easier to give feedback on a PR, because then I can do inline comments, etc. You can open a draft PR to indicate that it's not ready to be integrated.