Gen2NF now auto-scales bins to adapt to various volume levels. Gen2DFT had some cleanup and magic number formulas replaced by actual math I derived for the paper. The debug view scales bar intensity by value, which makes for nicer visualization.

Author: CaiB

ColorChord.NET/NoteFinder/Gen2NoteFinder.cs

@@ -3,7 +3,6 @@
 using ColorChord.NET.API.NoteFinder;
 using ColorChord.NET.API.Sources;
 using ColorChord.NET.Config;
-using ColorChord.NET.Outputs;
 using System;
 using System.Collections.Generic;
 using System.Diagnostics;
@@ -46,6 +45,10 @@ public sealed class Gen2NoteFinder : NoteFinderCommon, ITimingSource
     /// <summary> The frequencies (in Hz) of each of the raw bins in <see cref="AllBinValues"/>. </summary>
     public ReadOnlySpan<float> BinFrequencies => this.DFT.RawBinFrequencies;
 
+    private float AllBinMax = 0.01F, AllBinMaxSmoothed = 0.01F;
+
+    public float[] AllBinValuesScaled;
+
     public byte[] PeakBits, WidebandBits;
     private byte[] AllowedBinWidths;
 
@@ -87,6 +90,7 @@ private void Reconfigure()
         AllowedBinWidths = new byte[BinCount];
         PreviousBinValues = new float[BinCount];
         RecentBinChanges = new float[BinCount];
+        AllBinValuesScaled = new float[BinCount];
 
         for (int i = 0; i < AllowedBinWidths.Length; i++) { AllowedBinWidths[i] = (byte)MathF.Max(4F, 1.5F * MathF.Ceiling(this.DFT.RawBinWidths[i])); }
     }
@@ -166,6 +170,7 @@ public override void UpdateOutputs()
         for (int i = 0; i < WidebandBitsPacked.Length; i++) { WidebandBitsPacked[i] = 0; }
 
         Debug.Assert(((RawBinValuesPadded.Length - 2) % 8) == 0, "Bin count must be a multiple of 8"); // TODO: Add non-SIMD version to avoid this
+        Vector256<float> BinMaxIntermediate = Vector256<float>.Zero;
         for (int Step = 0; Step <= RawBinValuesPadded.Length - 10; Step += 8)
         {
             Vector256<float> LeftValues = Vector256.LoadUnsafe(ref RawBinValuesPadded[Step]); // TODO: Is there a better way to do this?
@@ -191,6 +196,18 @@ public override void UpdateOutputs()
             Vector256<float> NewRecentChange = Avx.Add(Avx.Multiply(Vector256.Create(RECENT_BIN_CHANGE_IIR), OldRecentChange), Avx.Multiply(Vector256.Create(RecentBinChangeIIRb), AbsDiffFromPrev));
             NewRecentChange.StoreUnsafe(ref RecentBinChanges[Step]);
             MiddleValues.StoreUnsafe(ref PreviousBinValues[Step]);
+            BinMaxIntermediate = Avx.Max(BinMaxIntermediate, MiddleValues);
+        }
+        Vector128<float> BinMaxCondense1 = Sse.Max(BinMaxIntermediate.GetLower(), BinMaxIntermediate.GetUpper());
+        float NewAllBinMax = MathF.Max(MathF.Max(BinMaxCondense1[0], BinMaxCondense1[1]), MathF.Max(BinMaxCondense1[2], BinMaxCondense1[3]));
+        this.AllBinMaxSmoothed = this.AllBinMax < NewAllBinMax ? (this.AllBinMax * 0.8F) + (NewAllBinMax * 0.2F) : (this.AllBinMax * 0.995F) + (NewAllBinMax * 0.005F);
+        this.AllBinMax = Math.Max(0.01F, AllBinMaxSmoothed);
+
+        for (int Step = 0; Step < AllBinValuesScaled.Length; Step += 8)
+        {
+            Vector256<float> MiddleValues = Vector256.LoadUnsafe(ref RawBinValuesPadded[Step + 1]);
+            Vector256<float> Scaled = Avx.Divide(MiddleValues, Vector256.Create(this.AllBinMaxSmoothed * 2.2F));
+            Scaled.StoreUnsafe(ref AllBinValuesScaled[Step]);
         }
 
         for (int Outer = 0; Outer < PeakBitsPacked.Length; Outer++)

ColorChord.NET/NoteFinder/Gen2NoteFinderDFT.cs

@@ -17,13 +17,13 @@ namespace ColorChord.NET.NoteFinder;
 /// subscribed to periodic events. Then, when a visualizer or other output mechanism wants to use the data, it will call <see cref="Gen2NoteFinder.UpdateOutputs"/>
 /// which in turn will call <see cref="CalculateOutput"/>. This will merge together all of the data that has been received since the last output cycle,
 /// apply loudness correction, and format it into <see cref="AllBinValues"/> and <see cref="OctaveBinValues"/> for feature extraction in <see cref="Gen2NoteFinder.UpdateOutputs"/>.
-internal sealed class Gen2NoteFinderDFT
+public sealed class Gen2NoteFinderDFT
 {
     private const bool ENABLE_SIMD = true;
     private const int MIN_WINDOW_SIZE = 16; // At 48KHz, scaled for other sample rates
     private const uint MAX_WINDOW_SIZE = 6144; // At 48KHz, scaled for other sample rates
     private const uint ABSOLUTE_MAX_WINDOW_SIZE = 32768; // Cannot exceed this regardless of sample rate, otherwise the accumulators may overflow
-    private const uint USHORT_RANGE = ushort.MaxValue + 1;
+    private const uint USHORT_RANGE = ((uint)ushort.MaxValue) + 1;
     private const uint BINS_PER_OCTAVE = 24;
 
     private const ushort SINE_TABLE_90_OFFSET = 8;
@@ -127,7 +127,7 @@ internal sealed class Gen2NoteFinderDFT
 
     /// <summary> The frequency in Hz of each raw bin. </summary>
     /// <remarks> This isn't used in the main algorithm, only for pre-calculation of some data, and to present to the user. </remarks>
-    internal float[] RawBinFrequencies { get; init; }
+    public float[] RawBinFrequencies { get; init; }
 
     /// <summary> The range, in number of bins, of each bin's sensitivity to input signals. </summary>
     /// <remarks> A bin width of 2 would imply that this bin stops detecting anything in approximately the middle of the directly adjacent bin in both directions (a range of this bin's middle +/- 1.0 bins). </remarks>
@@ -172,22 +172,11 @@ internal sealed class Gen2NoteFinderDFT
         {
             float BinFrequency;
             uint ThisBufferSize;
-            //if (Bin == 0)
-            //{
-                //BinFrequency = 1.5F + (2F * ((float)Bin / BINS_PER_OCTAVE));
-                //ThisBufferSize = (uint)MathF.Round(WindowSizeForBinWidth(0.5F, SampleRate));
-                //ThisBufferSize = Math.Min(ABSOLUTE_MAX_WINDOW_SIZE, ThisBufferSize);
-            //    BinFrequency = 75F;
-            //    ThisBufferSize = RoundedWindowSizeForBinWidth(110F, BinFrequency, SampleRate);
-            //}
-            //else
-            {
-                float ThisOctaveStart = StartFrequency * MathF.Pow(2, Bin / BINS_PER_OCTAVE);
-                BinFrequency = CalculateNoteFrequency(StartFrequency, BINS_PER_OCTAVE, Bin);
-                float NextBinFrequency = CalculateNoteFrequency(StartFrequency, BINS_PER_OCTAVE, Bin + 2);
-                //float IdealWindowSize = WindowSizeForBinWidth(TopOctaveNextBinFreq - TopOctaveBinFreq); // TODO: Add scale factor to shift this from no overlap to -3dB point
-                ThisBufferSize = RoundedWindowSizeForBinWidth(NextBinFrequency - BinFrequency, BinFrequency, SampleRate);
-            }
+            float ThisOctaveStart = StartFrequency * MathF.Pow(2, Bin / BINS_PER_OCTAVE);
+            BinFrequency = CalculateNoteFrequency(StartFrequency, BINS_PER_OCTAVE, Bin);
+            float NextBinFrequency = CalculateNoteFrequency(StartFrequency, BINS_PER_OCTAVE, Bin + 2);
+            //float IdealWindowSize = WindowSizeForBinWidth(TopOctaveNextBinFreq - TopOctaveBinFreq); // TODO: Add scale factor to shift this from no overlap to -3dB point
+            ThisBufferSize = RoundedWindowSizeForBinWidth(NextBinFrequency - BinFrequency, BinFrequency, SampleRate);
 
             RawBinWidths[Bin] = MathF.Log2((BinFrequency + BinWidthAtWindowSize(ThisBufferSize, SampleRate)) / BinFrequency) * BinsPerOctave;
 
@@ -201,16 +190,8 @@ internal sealed class Gen2NoteFinderDFT
             SinTableStepSize[Bin].NCLeft = (ushort)Math.Round(StepSizeNCL);
             SinTableStepSize[Bin].NCRight = (ushort)Math.Round(StepSizeNCR);
 
-            //if (Bin > 0)
-            {
-                LoudnessCorrectionFactors[Bin] = GetLoudnessCorrection(BinFrequency, LoudnessCorrectionAmount);
-                SmoothingFactors[Bin] = MathF.Min(MathF.Max(0.1F, IIR_CONST - ((6000 - (int)ThisBufferSize) * 0.00010F)), IIR_CONST);
-            }
-            //else
-            //{
-            //    SmoothingFactors[Bin] = 1F;
-            //    LoudnessCorrectionFactors[Bin] = 0.5F;
-            //}
+            LoudnessCorrectionFactors[Bin] = LoudnessCorrectionAmount == 0F ? 1F : GetLoudnessCorrection(BinFrequency, LoudnessCorrectionAmount);
+            SmoothingFactors[Bin] = MathF.Min(MathF.Max(0.1F, IIR_CONST - ((6000 - (int)ThisBufferSize) * 0.00008F)), IIR_CONST);
         }
         MaxPresentWindowSize = MaxAudioBufferSize;
 
@@ -247,6 +228,12 @@ internal sealed class Gen2NoteFinderDFT
         if (!ENABLE_SIMD) { Log.Warn("SIMD has been compile-time disabled in this build. Expect significantly decreased efficiency."); }
 #pragma warning restore CS0162 // Unreachable code detected
         if (!Avx2.IsSupported) { Log.Warn("Your CPU does not appear to support AVX2, this may lead to poor performance."); }
+#else
+        //Console.WriteLine("SSE? " + Sse.IsSupported);
+        //Console.WriteLine("SSE2? " + Sse2.IsSupported);
+        //Console.WriteLine("SSE4.2? " + Sse42.IsSupported);
+        //Console.WriteLine("AVX? " + Avx.IsSupported);
+        //Console.WriteLine("AVX2? " + Avx2.IsSupported);
 #endif
     }
 
@@ -637,6 +624,23 @@ public void CalculateOutput()
         }
     }
 
+    /// <summary> Clears all data structures, resetting back to a state equivalent to not having received any data yet. </summary>
+    public void Clear()
+    {
+        for (int i = 0; i < AudioBuffer.Length; i++) { AudioBuffer[i] = 0; }
+        for (int i = 0; i < BinCount; i++)
+        {
+            SinProductAccumulators[i] = new();
+            CosProductAccumulators[i] = new();
+            ProductAccumulators[i] = Vector256<long>.Zero;
+            MergedSinOutputs[i] = 0;
+            MergedCosOutputs[i] = 0;
+            AllBinValues[i + 1] = 0;
+            RawBinMagnitudes[i] = 0;
+        }
+        for (int i = 0; i < BinsPerOctave; i++) { OctaveBinValues[i] = 0; }
+    }
+
     /// <summary> Calculates the sine value of 2 positions </summary>
     /// <remarks> Does not require any SIMD support, however is significantly slower than <see cref="GetSine256(Vector256{ushort})"/> </remarks>
     /// <param name="sineTablePosition"> The wave positions, where one full sweep through possible values gives 1 full wavelength </param>
@@ -710,10 +714,8 @@ public struct DualI64
         public override readonly string ToString() => $"2xI32 L={this.NCLeft}, R={this.NCRight}";
     }
 
-    // Everyone loves magic numbers :)
-    // These were determined through simulations and regressions, which can be found in the Simulations folder in the root of the ColorChord.NET repository. See Simulations/WindowSizeVsBinWidthWithSampleRate.m.
-    private static float BinWidthAtWindowSize(float windowSize, float sampleRate) => ((sampleRate * 1.0414032F) + 592.49176F) / (windowSize + ((sampleRate * 0.00022935479F) + 1.5730453F));
-    private static float WindowSizeForBinWidth(float binWidth, float sampleRate) => (((sampleRate * 1.0414032F) + 592.49176F) / binWidth) - ((sampleRate * 0.00022935479F) + 1.5730453F);
+    private static float BinWidthAtWindowSize(float windowSize, float sampleRate) => sampleRate / windowSize;
+    private static float WindowSizeForBinWidth(float binWidth, float sampleRate) => sampleRate / binWidth;
 
     private static uint RoundedWindowSizeForBinWidth(float binWidth, float frequency, float sampleRate)
     {

ColorChord.NET/Outputs/Display/Shaders/ShinNFDebug.frag

@@ -36,9 +36,9 @@ void main()
     float ChangeHere = texture(TextureRawBins, vec2((SectionHere + 0.5) / BinCount, 0.75)).r * 10.0;
     uint PeakHere = (texture(TexturePeakBits, vec2(((SectionHere / 8) + 0.5) / textureSize(TexturePeakBits, 0).x, 0.5)).r >> (SectionHere % 8)) & 1u;
     uint WidebandHere = (texture(TextureWidebandBits, vec2(((SectionHere / 8) + 0.5) / textureSize(TextureWidebandBits, 0).x, 0.5)).r >> (SectionHere % 8)) & 1u;
-    vec3 Colour = AngleToRGB(mod(float(SectionHere) / BINS_PER_OCATVE, 1.0), 1.0 - (0.6 * WidebandHere), 1.0 - (0.8 * WidebandHere));
+    vec3 Colour = AngleToRGB(mod(float(SectionHere) / BINS_PER_OCATVE, 1.0), 1.0 - (0.8 * WidebandHere), min(1.0, 0.1 + (HeightHere * 3.0)) - (0.9 * WidebandHere));
     float IsBar = step(abs(TexCoord.y), HeightHere);
     //float IsBar = (step(0.0, TexCoord.y) * step(TexCoord.y, HeightHere)) + ((step(0.0, -TexCoord.y) * step(-TexCoord.y, ChangeHere)));
-    FragColor = vec4((IsBar * Colour) + ((1.0 - IsBar) * vec3(0.2) * PeakHere * Colour), 1.0);
+    FragColor = vec4((IsBar * Colour) + ((1.0 - IsBar) * vec3(0.1) * PeakHere * Colour), 1.0);
     //FragColor = vec4(vec3(0.8), 1.0);
 }

ColorChord.NET/Outputs/Display/ShinNFDebug.cs

@@ -120,15 +120,17 @@ public void Render()
         this.Shader!.Use();
 
         if (this.RawDataIn.Length != this.NoteFinder.AllBinValues.Length * 2) { this.RawDataIn = new float[this.NoteFinder.AllBinValues.Length * 2]; }
-        this.NoteFinder.AllBinValues.CopyTo(this.RawDataIn);
         if (this.G2NoteFinder != null)
         {
+            this.G2NoteFinder.AllBinValuesScaled.CopyTo(this.RawDataIn.AsSpan());
             this.G2NoteFinder.RecentBinChanges.CopyTo(this.RawDataIn, this.NoteFinder.AllBinValues.Length);
             GL.ActiveTexture(TextureUnit.Texture1);
             GL.TexImage2D(TextureTarget.Texture2D, 0, PixelInternalFormat.R8ui, this.NoteFinder.AllBinValues.Length / 8, 1, 0, PixelFormat.RedInteger, PixelType.UnsignedByte, this.G2NoteFinder.PeakBits);
             GL.ActiveTexture(TextureUnit.Texture2);
             GL.TexImage2D(TextureTarget.Texture2D, 0, PixelInternalFormat.R8ui, this.NoteFinder.AllBinValues.Length / 8, 1, 0, PixelFormat.RedInteger, PixelType.UnsignedByte, this.G2NoteFinder.WidebandBits);
         }
+        else { this.NoteFinder.AllBinValues.CopyTo(this.RawDataIn); }
+
         GL.ActiveTexture(TextureUnit.Texture0);
         GL.TexImage2D(TextureTarget.Texture2D, 0, PixelInternalFormat.R32f, this.NoteFinder.AllBinValues.Length, 2, 0, PixelFormat.Red, PixelType.Float, this.RawDataIn);
         GL.BindVertexArray(this.VertexArrayHandle);

Gen2DFTLib/Gen2DFTLib.csproj

@@ -8,6 +8,8 @@
 	<PublishAot>true</PublishAot>
 	<DefineConstants>STANDALONE_DFT_LIB</DefineConstants>
 	<AllowUnsafeBlocks>True</AllowUnsafeBlocks>
+	<IlcInstructionSet>native</IlcInstructionSet>
+	<IlcDisableReflection>true</IlcDisableReflection>
   </PropertyGroup>
 
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|AnyCPU'">

Gen2DFTLib/Properties/PublishProfiles/FolderProfile.pubxml

@@ -0,0 +1,18 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!--
+https://go.microsoft.com/fwlink/?LinkID=208121.
+-->
+<Project>
+  <PropertyGroup>
+    <Configuration>Release</Configuration>
+    <Platform>x64</Platform>
+    <PublishDir>bin\AOT\</PublishDir>
+    <PublishProtocol>FileSystem</PublishProtocol>
+    <_TargetId>Folder</_TargetId>
+    <TargetFramework>net8.0</TargetFramework>
+    <RuntimeIdentifier>win-x64</RuntimeIdentifier>
+    <SelfContained>true</SelfContained>
+    <PublishSingleFile>false</PublishSingleFile>
+    <PublishReadyToRun>false</PublishReadyToRun>
+  </PropertyGroup>
+</Project>