From ba5de2b705e82975a48e227b41ba48872478c1bf Mon Sep 17 00:00:00 2001 From: Tarek Date: Thu, 21 Dec 2023 13:53:40 +0100 Subject: [PATCH 1/5] use mel scale for frequencies --- src/lenses/SpectrogramLens/index.tsx | 39 ++++++++++++++-------------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/src/lenses/SpectrogramLens/index.tsx b/src/lenses/SpectrogramLens/index.tsx index ff22e17a..eeb4eb50 100644 --- a/src/lenses/SpectrogramLens/index.tsx +++ b/src/lenses/SpectrogramLens/index.tsx @@ -41,6 +41,8 @@ const drawScale = ( numTicks = Math.round(height / 20); } else if (scale === 'linear') { numTicks = Math.round(height / 30); + } else if (scale === 'mel') { + numTicks = Math.round(height / 30); } else { numTicks = 5; } @@ -60,6 +62,19 @@ const drawScale = ( .ticks(numTicks, (x: number) => { return `${freqType(x).toFixed(1)} ${unitType(x)}`; }); + } else if (scale === 'mel') { + const domain = [LOG_DOMAIN_LOWER_LIMIT, upperLimit]; + const range = [height, 0]; + const scale = d3.scaleLog(domain, range); + + axis = d3 + .axisRight(scale) + .scale(scale) + .tickPadding(1) + .tickSizeOuter(0) + .ticks(numTicks, (x: number) => { + return `${hzToMel(x).toFixed(1)}`; + }); } else { const domain = [upperLimit, 0]; const range = [0, height]; @@ -241,24 +256,6 @@ const SpectrogramLens: Lens = ({ columns, urls, values }) => { } } - drawData[i] = col; - } - } else if (ampScale === 'mel') { - for (let i = 0; i < frequenciesData.length; i++) { - const col = []; - - for (let j = 0; j < frequenciesData[i].length; j++) { - const amplitude = frequenciesData[i][j]; - col[j] = hzToMel(amplitude ** 2); - - if (col[j] > max) { - max = col[j]; - } - - if (col[j] < min) { - min = col[j]; - } - } drawData[i] = col; } } else { @@ -285,6 +282,8 @@ const SpectrogramLens: Lens = ({ columns, urls, values }) => { value = heightScale(scaleFunc.invert(height - y)); } else if (freqScale === 'linear') { value = Math.abs(heightScale(height - y)); + } else if (freqScale === 'mel') { + value = hzToMel(scaleFunc.invert(height - y)); } const indexA = Math.floor(value); @@ -464,10 +463,10 @@ const SpectrogramLens: Lens = ({ columns, urls, values }) => { } From 2d2fe033b8b42aff9ce6a84716787aa6d3e48b94 Mon Sep 17 00:00:00 2001 From: Tarek Date: Thu, 21 Dec 2023 13:53:40 +0100 Subject: [PATCH 2/5] create mel scale from d3 scale --- src/lenses/SpectrogramLens/index.tsx | 89 ++++++++++++++++++++++++---- 1 file changed, 79 insertions(+), 10 deletions(-) diff --git a/src/lenses/SpectrogramLens/index.tsx b/src/lenses/SpectrogramLens/index.tsx index eeb4eb50..7a7fa28b 100644 --- a/src/lenses/SpectrogramLens/index.tsx +++ b/src/lenses/SpectrogramLens/index.tsx @@ -10,7 +10,7 @@ import { ColorsState, useColors } from '../../stores/colors'; import { Lens } from '../../types'; import useSetting from '../useSetting'; import MenuBar from './MenuBar'; -import { fixWindow, freqType, unitType, amplitudeToDb, hzToMel } from './Spectrogram'; +import { fixWindow, freqType, unitType, amplitudeToDb, melToHz } from './Spectrogram'; const Container = tw.div`flex flex-col w-full h-full items-stretch justify-center`; const EmptyNote = styled.p` @@ -25,6 +25,70 @@ interface WebAudio_ extends WebAudio { const LOG_DOMAIN_LOWER_LIMIT = 10; const FFT_SAMPLES = 1024; +interface MelScale { + (value: number): number; + domain(): number[]; + domain(domain: number[]): MelScale; + range(): number[]; + range(range: number[]): MelScale; + copy(): MelScale; + invert(value: number): number; + ticks(count?: number): number[]; + tickFormat(count?: number, specifier?: string): (d: number) => string; +} +function toMelScale(frequency: number): number { + return 2595 * Math.log10(1 + frequency / 700); +} + +function fromMelScale(mel: number): number { + return 700 * (Math.pow(10, mel / 2595) - 1); +} + +function melScale(): MelScale { + // Create the base log scale + const linearScale = d3.scaleLinear(); + + // Our custom scale function + const scale: MelScale = ((value: number) => { + return linearScale(value); + }) as MelScale; + + // Copy methods from the log scale + // eslint-disable-next-line @typescript-eslint/no-explicit-any + scale.domain = (domain?: number[]): any => { + if (domain === undefined) { + return linearScale.domain().map((d) => toMelScale(d)); + } + return domain ? (linearScale.domain(domain), scale) : linearScale.domain(); + }; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + scale.range = (range?: number[]): any => { + return range ? (linearScale.range(range), scale) : linearScale.range(); + }; + scale.copy = () => { + return melScale().domain(scale.domain()).range(scale.range()); + }; + scale.invert = (value: number): number => { + return melToHz(linearScale.invert(value)); + }; + + scale.ticks = (count?: number): number[] => { + const ticks = linearScale.ticks(count).map((num) => { + return toMelScale(num); + }); + return ticks; + }; + + scale.tickFormat = ( + count?: number, + specifier?: string + ): ((d: number) => string) => { + return linearScale.tickFormat(count, specifier); + }; + + return scale; +} + /* * Redraws the scale for resized spectrogram */ @@ -63,18 +127,22 @@ const drawScale = ( return `${freqType(x).toFixed(1)} ${unitType(x)}`; }); } else if (scale === 'mel') { - const domain = [LOG_DOMAIN_LOWER_LIMIT, upperLimit]; - const range = [height, 0]; - const scale = d3.scaleLog(domain, range); + const domain: [number, number] = [20, upperLimit]; + const range: [number, number] = [height, 0]; + const scale = melScale().domain(domain).range(range); axis = d3 .axisRight(scale) .scale(scale) .tickPadding(1) .tickSizeOuter(0) - .ticks(numTicks, (x: number) => { - return `${hzToMel(x).toFixed(1)}`; - }); + .ticks(numTicks) + .tickFormat( + (x: number) => + `${freqType(fromMelScale(x).valueOf()).toFixed(1)} ${unitType( + fromMelScale(x).valueOf() + )}` + ); } else { const domain = [upperLimit, 0]; const range = [0, height]; @@ -209,8 +277,6 @@ const SpectrogramLens: Lens = ({ columns, urls, values }) => { const imageData = new ImageData(width, height); - const scaleFunc = d3.scaleLog(domain, range); - // Default to linear scale let heightScale = d3.scaleLinear(domain, [0, upperLimit]); @@ -275,6 +341,8 @@ const SpectrogramLens: Lens = ({ columns, urls, values }) => { } const colorScale = colorPalette.scale().domain([min, max]); + const scaleFunc = d3.scaleLog(domain, range); + for (let y = 0; y < height; y++) { let value = 0; @@ -283,7 +351,8 @@ const SpectrogramLens: Lens = ({ columns, urls, values }) => { } else if (freqScale === 'linear') { value = Math.abs(heightScale(height - y)); } else if (freqScale === 'mel') { - value = hzToMel(scaleFunc.invert(height - y)); + value = + scaleFunc(fromMelScale(height)) - scaleFunc(fromMelScale(y)); } const indexA = Math.floor(value); From acaf15988541c44596bd44352974a52fc39a2c07 Mon Sep 17 00:00:00 2001 From: Tarek Date: Fri, 8 Mar 2024 14:43:49 +0100 Subject: [PATCH 3/5] fix mel scale --- src/lenses/SpectrogramLens/index.tsx | 67 ++++++++++++++++++---------- 1 file changed, 44 insertions(+), 23 deletions(-) diff --git a/src/lenses/SpectrogramLens/index.tsx b/src/lenses/SpectrogramLens/index.tsx index 7a7fa28b..0ce1e686 100644 --- a/src/lenses/SpectrogramLens/index.tsx +++ b/src/lenses/SpectrogramLens/index.tsx @@ -10,7 +10,7 @@ import { ColorsState, useColors } from '../../stores/colors'; import { Lens } from '../../types'; import useSetting from '../useSetting'; import MenuBar from './MenuBar'; -import { fixWindow, freqType, unitType, amplitudeToDb, melToHz } from './Spectrogram'; +import { fixWindow, freqType, unitType, amplitudeToDb } from './Spectrogram'; const Container = tw.div`flex flex-col w-full h-full items-stretch justify-center`; const EmptyNote = styled.p` @@ -22,7 +22,7 @@ interface WebAudio_ extends WebAudio { buffer: AudioBuffer; } -const LOG_DOMAIN_LOWER_LIMIT = 10; +const DOMAIN_LOWER_LIMIT = 10; const FFT_SAMPLES = 1024; interface MelScale { @@ -36,6 +36,7 @@ interface MelScale { ticks(count?: number): number[]; tickFormat(count?: number, specifier?: string): (d: number) => string; } + function toMelScale(frequency: number): number { return 2595 * Math.log10(1 + frequency / 700); } @@ -47,6 +48,7 @@ function fromMelScale(mel: number): number { function melScale(): MelScale { // Create the base log scale const linearScale = d3.scaleLinear(); + const logScale = d3.scaleLog(); // Our custom scale function const scale: MelScale = ((value: number) => { @@ -59,31 +61,28 @@ function melScale(): MelScale { if (domain === undefined) { return linearScale.domain().map((d) => toMelScale(d)); } + logScale.domain(domain); return domain ? (linearScale.domain(domain), scale) : linearScale.domain(); }; // eslint-disable-next-line @typescript-eslint/no-explicit-any scale.range = (range?: number[]): any => { + if (range) logScale.range(range); return range ? (linearScale.range(range), scale) : linearScale.range(); }; scale.copy = () => { return melScale().domain(scale.domain()).range(scale.range()); }; scale.invert = (value: number): number => { - return melToHz(linearScale.invert(value)); + return fromMelScale(linearScale.invert(value)); }; scale.ticks = (count?: number): number[] => { - const ticks = linearScale.ticks(count).map((num) => { - return toMelScale(num); + const ticks = logScale.ticks(count).map((val) => { + return toMelScale(val); }); - return ticks; - }; + if (!count) return ticks; - scale.tickFormat = ( - count?: number, - specifier?: string - ): ((d: number) => string) => { - return linearScale.tickFormat(count, specifier); + return [ticks[0]].concat(ticks.slice(ticks.length - count, ticks.length)); }; return scale; @@ -106,7 +105,7 @@ const drawScale = ( } else if (scale === 'linear') { numTicks = Math.round(height / 30); } else if (scale === 'mel') { - numTicks = Math.round(height / 30); + numTicks = Math.round(height / 40); } else { numTicks = 5; } @@ -114,7 +113,7 @@ const drawScale = ( let axis; if (scale === 'logarithmic') { - const domain = [LOG_DOMAIN_LOWER_LIMIT, upperLimit]; + const domain = [DOMAIN_LOWER_LIMIT, upperLimit]; const range = [height, 0]; const scale = d3.scaleLog(domain, range); @@ -127,7 +126,7 @@ const drawScale = ( return `${freqType(x).toFixed(1)} ${unitType(x)}`; }); } else if (scale === 'mel') { - const domain: [number, number] = [20, upperLimit]; + const domain: [number, number] = [DOMAIN_LOWER_LIMIT, upperLimit]; const range: [number, number] = [height, 0]; const scale = melScale().domain(domain).range(range); @@ -270,7 +269,7 @@ const SpectrogramLens: Lens = ({ columns, urls, values }) => { const upperLimit = backend.buffer?.sampleRate / 2; // 10, ..., half-samplerate (default 22050) - const domain = [LOG_DOMAIN_LOWER_LIMIT, upperLimit]; + const domain = [DOMAIN_LOWER_LIMIT, upperLimit]; // 0, ..., canvas-height const range = [0, height]; @@ -283,7 +282,9 @@ const SpectrogramLens: Lens = ({ columns, urls, values }) => { if (freqScale === 'logarithmic') { heightScale = d3.scaleLinear(domain, [0, FFT_SAMPLES / 2 - 1]); } else if (freqScale === 'linear') { - heightScale = d3.scaleLinear(domain, [0, upperLimit]); + heightScale = d3.scaleLinear(range, [0, FFT_SAMPLES / 2]); + } else if (freqScale === 'mel') { + heightScale = d3.scaleLinear(domain, [0, FFT_SAMPLES / 2 - 1]); } const widthScale = d3.scaleLinear([0, width], [0, frequenciesData.length]); @@ -302,7 +303,6 @@ const SpectrogramLens: Lens = ({ columns, urls, values }) => { ref = maxI; } } - //const top_db = 80; const amin = 1e-5; // Convert amplitudes to decibels @@ -349,10 +349,13 @@ const SpectrogramLens: Lens = ({ columns, urls, values }) => { if (freqScale === 'logarithmic') { value = heightScale(scaleFunc.invert(height - y)); } else if (freqScale === 'linear') { - value = Math.abs(heightScale(height - y)); + value = heightScale(height - y); } else if (freqScale === 'mel') { - value = - scaleFunc(fromMelScale(height)) - scaleFunc(fromMelScale(y)); + const scaleFunc = melScale() + .domain([DOMAIN_LOWER_LIMIT, toMelScale(upperLimit)]) + .range(range); + heightScale = d3.scaleLinear([0, upperLimit], [0, FFT_SAMPLES / 2]); + value = heightScale(scaleFunc.invert(height - y)); } const indexA = Math.floor(value); @@ -456,7 +459,7 @@ const SpectrogramLens: Lens = ({ columns, urls, values }) => { const coords = d3.pointer(e); if (freqScale === 'logarithmic') { - const domain = [LOG_DOMAIN_LOWER_LIMIT, upperLimit]; + const domain = [DOMAIN_LOWER_LIMIT, upperLimit]; const range = [0, height]; const scale = d3.scaleLog(domain, range); @@ -467,6 +470,24 @@ const SpectrogramLens: Lens = ({ columns, urls, values }) => { .attr('y', coords[1]) .attr('font-size', 10) .attr('fill', theme`colors.white`); + } else if (freqScale === 'mel') { + let y = height - coords[1]; + + const domain: [number, number] = [ + DOMAIN_LOWER_LIMIT, + toMelScale(upperLimit), + ]; + const range: [number, number] = [0, height]; + + const scaleFunc = melScale().domain(domain).range(range); + y = scaleFunc.invert(y); + + d3.select(mouseLabel.current) + .text(`${freqType(y).toFixed(1)} ${unitType(y)}`) + .attr('x', coords[0]) + .attr('y', coords[1]) + .attr('font-size', 10) + .attr('fill', theme`colors.white`); } else { //if (freqScale === 'linear') { const domain = [upperLimit, 0]; @@ -526,11 +547,11 @@ const SpectrogramLens: Lens = ({ columns, urls, values }) => {
+ wiki { // Add the menubar as last component, so that it is rendered on top // We don't use a z-index for this, because it interferes with the rendering of the contained menus } - Date: Thu, 21 Dec 2023 13:53:40 +0100 Subject: [PATCH 4/5] use mel scale for frequencies --- src/lenses/SpectrogramLens/index.tsx | 39 ++++++++++++++-------------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/src/lenses/SpectrogramLens/index.tsx b/src/lenses/SpectrogramLens/index.tsx index ff22e17a..eeb4eb50 100644 --- a/src/lenses/SpectrogramLens/index.tsx +++ b/src/lenses/SpectrogramLens/index.tsx @@ -41,6 +41,8 @@ const drawScale = ( numTicks = Math.round(height / 20); } else if (scale === 'linear') { numTicks = Math.round(height / 30); + } else if (scale === 'mel') { + numTicks = Math.round(height / 30); } else { numTicks = 5; } @@ -60,6 +62,19 @@ const drawScale = ( .ticks(numTicks, (x: number) => { return `${freqType(x).toFixed(1)} ${unitType(x)}`; }); + } else if (scale === 'mel') { + const domain = [LOG_DOMAIN_LOWER_LIMIT, upperLimit]; + const range = [height, 0]; + const scale = d3.scaleLog(domain, range); + + axis = d3 + .axisRight(scale) + .scale(scale) + .tickPadding(1) + .tickSizeOuter(0) + .ticks(numTicks, (x: number) => { + return `${hzToMel(x).toFixed(1)}`; + }); } else { const domain = [upperLimit, 0]; const range = [0, height]; @@ -241,24 +256,6 @@ const SpectrogramLens: Lens = ({ columns, urls, values }) => { } } - drawData[i] = col; - } - } else if (ampScale === 'mel') { - for (let i = 0; i < frequenciesData.length; i++) { - const col = []; - - for (let j = 0; j < frequenciesData[i].length; j++) { - const amplitude = frequenciesData[i][j]; - col[j] = hzToMel(amplitude ** 2); - - if (col[j] > max) { - max = col[j]; - } - - if (col[j] < min) { - min = col[j]; - } - } drawData[i] = col; } } else { @@ -285,6 +282,8 @@ const SpectrogramLens: Lens = ({ columns, urls, values }) => { value = heightScale(scaleFunc.invert(height - y)); } else if (freqScale === 'linear') { value = Math.abs(heightScale(height - y)); + } else if (freqScale === 'mel') { + value = hzToMel(scaleFunc.invert(height - y)); } const indexA = Math.floor(value); @@ -464,10 +463,10 @@ const SpectrogramLens: Lens = ({ columns, urls, values }) => { } From 008c5ff7056eb6f4228fd6521374d17a4db06d03 Mon Sep 17 00:00:00 2001 From: Tarek Date: Fri, 8 Mar 2024 15:03:25 +0100 Subject: [PATCH 5/5] outsource melscale to different file --- src/lenses/SpectrogramLens/MelScale.tsx | 68 +++++++++++++++++++++++ src/lenses/SpectrogramLens/index.tsx | 72 ++----------------------- 2 files changed, 73 insertions(+), 67 deletions(-) create mode 100644 src/lenses/SpectrogramLens/MelScale.tsx diff --git a/src/lenses/SpectrogramLens/MelScale.tsx b/src/lenses/SpectrogramLens/MelScale.tsx new file mode 100644 index 00000000..471364da --- /dev/null +++ b/src/lenses/SpectrogramLens/MelScale.tsx @@ -0,0 +1,68 @@ +import * as d3 from 'd3'; + +interface MelScale { + (value: number): number; + toMelScale(value: number): number; + fromMelScale(frequency: number): number; + domain(): number[]; + domain(domain: number[]): MelScale; + range(): number[]; + range(range: number[]): MelScale; + copy(): MelScale; + invert(value: number): number; + ticks(count?: number): number[]; + tickFormat(count?: number, specifier?: string): (d: number) => string; +} + +const melScale = (): MelScale => { + // Create the base log scale + const linearScale = d3.scaleLinear(); + const logScale = d3.scaleLog(); + + // Our custom scale function + const scale: MelScale = ((value: number) => { + return linearScale(value); + }) as MelScale; + + scale.toMelScale = (frequency: number): number => { + return 2595 * Math.log10(1 + frequency / 700); + }; + + scale.fromMelScale = (mel: number): number => { + return 700 * (Math.pow(10, mel / 2595) - 1); + }; + + // Copy methods from the log scale + // eslint-disable-next-line @typescript-eslint/no-explicit-any + scale.domain = (domain?: number[]): any => { + if (domain === undefined) { + return linearScale.domain().map((d) => scale.toMelScale(d)); + } + logScale.domain(domain); + return domain ? (linearScale.domain(domain), scale) : linearScale.domain(); + }; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + scale.range = (range?: number[]): any => { + if (range) logScale.range(range); + return range ? (linearScale.range(range), scale) : linearScale.range(); + }; + scale.copy = () => { + return melScale().domain(scale.domain()).range(scale.range()); + }; + scale.invert = (value: number): number => { + return scale.fromMelScale(linearScale.invert(value)); + }; + + scale.ticks = (count?: number): number[] => { + const ticks = logScale.ticks(count).map((val) => { + return scale.toMelScale(val); + }); + if (!count) return ticks; + + return [ticks[0]].concat(ticks.slice(ticks.length - count, ticks.length)); + }; + + return scale; +}; + +export default melScale; diff --git a/src/lenses/SpectrogramLens/index.tsx b/src/lenses/SpectrogramLens/index.tsx index 4ee705a0..cf7197bc 100644 --- a/src/lenses/SpectrogramLens/index.tsx +++ b/src/lenses/SpectrogramLens/index.tsx @@ -10,6 +10,7 @@ import { ColorsState, useColors } from '../../stores/colors'; import { Lens } from '../../types'; import useSetting from '../useSetting'; import MenuBar from './MenuBar'; +import melScale from './MelScale'; import { fixWindow, freqType, unitType, amplitudeToDb } from './Spectrogram'; const Container = tw.div`flex flex-col w-full h-full items-stretch justify-center`; @@ -25,69 +26,6 @@ interface WebAudio_ extends WebAudio { const DOMAIN_LOWER_LIMIT = 10; const FFT_SAMPLES = 1024; -interface MelScale { - (value: number): number; - domain(): number[]; - domain(domain: number[]): MelScale; - range(): number[]; - range(range: number[]): MelScale; - copy(): MelScale; - invert(value: number): number; - ticks(count?: number): number[]; - tickFormat(count?: number, specifier?: string): (d: number) => string; -} - -function toMelScale(frequency: number): number { - return 2595 * Math.log10(1 + frequency / 700); -} - -function fromMelScale(mel: number): number { - return 700 * (Math.pow(10, mel / 2595) - 1); -} - -function melScale(): MelScale { - // Create the base log scale - const linearScale = d3.scaleLinear(); - const logScale = d3.scaleLog(); - - // Our custom scale function - const scale: MelScale = ((value: number) => { - return linearScale(value); - }) as MelScale; - - // Copy methods from the log scale - // eslint-disable-next-line @typescript-eslint/no-explicit-any - scale.domain = (domain?: number[]): any => { - if (domain === undefined) { - return linearScale.domain().map((d) => toMelScale(d)); - } - logScale.domain(domain); - return domain ? (linearScale.domain(domain), scale) : linearScale.domain(); - }; - // eslint-disable-next-line @typescript-eslint/no-explicit-any - scale.range = (range?: number[]): any => { - if (range) logScale.range(range); - return range ? (linearScale.range(range), scale) : linearScale.range(); - }; - scale.copy = () => { - return melScale().domain(scale.domain()).range(scale.range()); - }; - scale.invert = (value: number): number => { - return fromMelScale(linearScale.invert(value)); - }; - - scale.ticks = (count?: number): number[] => { - const ticks = logScale.ticks(count).map((val) => { - return toMelScale(val); - }); - if (!count) return ticks; - - return [ticks[0]].concat(ticks.slice(ticks.length - count, ticks.length)); - }; - - return scale; -} - /* * Redraws the scale for resized spectrogram */ @@ -138,8 +76,8 @@ const drawScale = ( .ticks(numTicks) .tickFormat( (x: number) => - `${freqType(fromMelScale(x).valueOf()).toFixed(1)} ${unitType( - fromMelScale(x).valueOf() + `${freqType(scale.fromMelScale(x).valueOf()).toFixed(1)} ${unitType( + scale.fromMelScale(x).valueOf() )}` ); } else { @@ -352,7 +290,7 @@ const SpectrogramLens: Lens = ({ columns, urls, values }) => { value = heightScale(height - y); } else if (freqScale === 'mel') { const scaleFunc = melScale() - .domain([DOMAIN_LOWER_LIMIT, toMelScale(upperLimit)]) + .domain([DOMAIN_LOWER_LIMIT, melScale().toMelScale(upperLimit)]) .range(range); heightScale = d3.scaleLinear([0, upperLimit], [0, FFT_SAMPLES / 2]); value = heightScale(scaleFunc.invert(height - y)); @@ -475,7 +413,7 @@ const SpectrogramLens: Lens = ({ columns, urls, values }) => { const domain: [number, number] = [ DOMAIN_LOWER_LIMIT, - toMelScale(upperLimit), + melScale().toMelScale(upperLimit), ]; const range: [number, number] = [0, height];