Work on Column performance (#13074)

Following the hit from adding progress logging attempting to improve it.

- Only do a progress message every 50,000 rows (still probably too frequent).
- Tuned `ProgressHandler` to count down to avoid any division in step check.
- Separated the specialized iterators from standard java iterators on `Storage`, making it clearer to see when using them.
- Removed the `zip` method from primitive iterators as when testing found for loop accessing boxed values was significantly quicker.
- Tuned map and build to have two cleaner and simpler methods - one preserving nulls and one passing nulls. Boxed was slower than primitive in testing here.

**Running Perf test on develop and this to compare results.**

Author: jdunkerley

std-bits/table/src/main/java/org/enso/table/data/column/operation/StorageIterators.java

@@ -323,7 +323,7 @@ public Storage<Boolean> slice(List<SliceRange> ranges) {
   }
 
   @Override
-  public ColumnBooleanStorageIterator iterator() {
+  public ColumnBooleanStorageIterator iteratorWithIndex() {
     return new BoolStorageIterator(this);
   }
 

std-bits/table/src/main/java/org/enso/table/data/column/storage/BoolStorage.java

@@ -5,5 +5,5 @@ public interface ColumnBooleanStorage extends ColumnStorage<Boolean> {
   boolean getItemAsBoolean(long index) throws ValueIsNothingException;
 
   @Override
-  ColumnBooleanStorageIterator iterator();
+  ColumnBooleanStorageIterator iteratorWithIndex();
 }

std-bits/table/src/main/java/org/enso/table/data/column/storage/ColumnBooleanStorage.java

@@ -3,28 +3,4 @@
 public interface ColumnBooleanStorageIterator extends ColumnStorageIterator<Boolean> {
   /** Gets the current item as a boolean. Note if the item isNothing value is undefined. */
   boolean getItemAsBoolean();
-
-  @FunctionalInterface
-  interface BooleanBooleanZipper {
-    void accept(long idx, boolean value1, boolean isNothing1, boolean value2, boolean isNothing2);
-  }
-
-  /** Zips this iterator with another iterator. */
-  default void zip(
-      ColumnBooleanStorage otherStorage, ColumnBooleanStorageIterator.BooleanBooleanZipper zipper) {
-    var other = otherStorage.iterator();
-
-    boolean hasValue1 = moveNext();
-    boolean hasValue2 = other.moveNext();
-    long idx = 0;
-    while (hasValue1 || hasValue2) {
-      boolean isNothing1 = !hasValue1 || isNothing();
-      boolean value1 = isNothing1 && getItemAsBoolean();
-      boolean isNothing2 = !hasValue2 || other.isNothing();
-      boolean value2 = isNothing2 && other.getItemAsBoolean();
-      zipper.accept(idx++, value1, isNothing1, value2, isNothing2);
-      hasValue1 = hasValue1 && moveNext();
-      hasValue2 = hasValue2 && other.moveNext();
-    }
-  }
 }

std-bits/table/src/main/java/org/enso/table/data/column/storage/ColumnBooleanStorageIterator.java

@@ -7,7 +7,7 @@ public interface ColumnDoubleStorage extends ColumnStorage<Double> {
   double getItemAsDouble(long index) throws ValueIsNothingException;
 
   @Override
-  ColumnDoubleStorageIterator iterator();
+  ColumnDoubleStorageIterator iteratorWithIndex();
 
   @Override
   FloatType getType();

std-bits/table/src/main/java/org/enso/table/data/column/storage/ColumnDoubleStorage.java

@@ -3,50 +3,4 @@
 public interface ColumnDoubleStorageIterator extends ColumnStorageIterator<Double> {
   /** Gets the current item as a double. Note if the item isNothing value is undefined. */
   double getItemAsDouble();
-
-  @FunctionalInterface
-  interface DoubleDoubleZipper {
-    void accept(long idx, double value1, boolean isNothing1, double value2, boolean isNothing2);
-  }
-
-  @FunctionalInterface
-  interface DoubleLongZipper {
-    void accept(long idx, double value1, boolean isNothing1, long value2, boolean isNothing2);
-  }
-
-  /** Zips this iterator with another iterator. */
-  default void zip(ColumnDoubleStorage otherStorage, DoubleDoubleZipper zipper) {
-    var other = otherStorage.iterator();
-
-    boolean hasValue1 = moveNext();
-    boolean hasValue2 = other.moveNext();
-    long idx = 0;
-    while (hasValue1 || hasValue2) {
-      boolean isNothing1 = !hasValue1 || isNothing();
-      double value1 = isNothing1 ? Double.NaN : getItemAsDouble();
-      boolean isNothing2 = !hasValue2 || other.isNothing();
-      double value2 = isNothing2 ? Double.NaN : other.getItemAsDouble();
-      zipper.accept(idx++, value1, isNothing1, value2, isNothing2);
-      hasValue1 = hasValue1 && moveNext();
-      hasValue2 = hasValue2 && other.moveNext();
-    }
-  }
-
-  /** Zips this iterator with another iterator. */
-  default void zip(ColumnLongStorage otherStorage, DoubleLongZipper zipper) {
-    var other = otherStorage.iterator();
-
-    boolean hasValue1 = moveNext();
-    boolean hasValue2 = other.moveNext();
-    long idx = 0;
-    while (hasValue1 || hasValue2) {
-      boolean isNothing1 = !hasValue1 || isNothing();
-      double value1 = isNothing1 ? Double.NaN : getItemAsDouble();
-      boolean isNothing2 = !hasValue2 || other.isNothing();
-      long value2 = isNothing2 ? 0 : other.getItemAsLong();
-      zipper.accept(idx++, value1, isNothing1, value2, isNothing2);
-      hasValue1 = hasValue1 && moveNext();
-      hasValue2 = hasValue2 && other.moveNext();
-    }
-  }
 }

std-bits/table/src/main/java/org/enso/table/data/column/storage/ColumnDoubleStorageIterator.java

@@ -7,7 +7,7 @@ public interface ColumnLongStorage extends ColumnStorage<Long> {
   long getItemAsLong(long index) throws ValueIsNothingException;
 
   @Override
-  ColumnLongStorageIterator iterator();
+  ColumnLongStorageIterator iteratorWithIndex();
 
   @Override
   IntegerType getType();

std-bits/table/src/main/java/org/enso/table/data/column/storage/ColumnLongStorage.java

@@ -3,50 +3,4 @@
 public interface ColumnLongStorageIterator extends ColumnStorageIterator<Long> {
   /** Gets the current item as a long. Note if the item isNothing value is undefined. */
   long getItemAsLong();
-
-  @FunctionalInterface
-  interface LongLongZipper {
-    void accept(long idx, long value1, boolean isNothing1, long value2, boolean isNothing2);
-  }
-
-  @FunctionalInterface
-  interface LongDoubleZipper {
-    void accept(long idx, long value1, boolean isNothing1, double value2, boolean isNothing2);
-  }
-
-  /** Zips this iterator with a Double storage. */
-  default void zip(ColumnDoubleStorage otherStorage, LongDoubleZipper zipper) {
-    var other = otherStorage.iterator();
-
-    boolean hasValue1 = moveNext();
-    boolean hasValue2 = other.moveNext();
-    long idx = 0;
-    while (hasValue1 || hasValue2) {
-      boolean isNothing1 = !hasValue1 || isNothing();
-      long value1 = isNothing1 ? 0 : getItemAsLong();
-      boolean isNothing2 = !hasValue2 || other.isNothing();
-      double value2 = isNothing2 ? 0 : other.getItemAsDouble();
-      zipper.accept(idx++, value1, isNothing1, value2, isNothing2);
-      hasValue1 = hasValue1 && moveNext();
-      hasValue2 = hasValue2 && other.moveNext();
-    }
-  }
-
-  /** Zips this iterator with another Long storage. */
-  default void zip(ColumnLongStorage otherStorage, LongLongZipper zipper) {
-    var other = otherStorage.iterator();
-
-    boolean hasValue1 = moveNext();
-    boolean hasValue2 = other.moveNext();
-    long idx = 0;
-    while (hasValue1 || hasValue2) {
-      boolean isNothing1 = !hasValue1 || isNothing();
-      long value1 = isNothing1 ? 0 : getItemAsLong();
-      boolean isNothing2 = !hasValue2 || other.isNothing();
-      long value2 = isNothing2 ? 0 : other.getItemAsLong();
-      zipper.accept(idx++, value1, isNothing1, value2, isNothing2);
-      hasValue1 = hasValue1 && moveNext();
-      hasValue2 = hasValue2 && other.moveNext();
-    }
-  }
 }

std-bits/table/src/main/java/org/enso/table/data/column/storage/ColumnLongStorageIterator.java

@@ -21,6 +21,6 @@ public interface ColumnStorage<T> extends Iterable<T> {
   /* Gets the value at a given index. */
   T getItemBoxed(long index);
 
-  @Override
-  ColumnStorageIterator<T> iterator();
+  /* Gets an iterator with index tracking. */
+  ColumnStorageIterator<T> iteratorWithIndex();
 }

std-bits/table/src/main/java/org/enso/table/data/column/storage/ColumnStorage.java

@@ -1,5 +1,6 @@
 package org.enso.table.data.column.storage;
 
+import java.util.Iterator;
 import java.util.function.Function;
 import org.enso.table.data.column.storage.type.StorageType;
 
@@ -35,7 +36,29 @@ public T getItemBoxed(long index) {
   }
 
   @Override
-  public ColumnStorageIterator<T> iterator() {
+  public Iterator<T> iterator() {
+    return new Iterator<T>() {
+      private final Iterator<S> parentIterator = parent.iterator();
+
+      @Override
+      public boolean hasNext() {
+        return parentIterator.hasNext();
+      }
+
+      @Override
+      public T next() {
+        S item = parentIterator.next();
+        if (item == null) {
+          return null;
+        } else {
+          return converter.apply(item);
+        }
+      }
+    };
+  }
+
+  @Override
+  public ColumnStorageIterator<T> iteratorWithIndex() {
     return new Storage.StorageIterator<>(this);
   }
 }