From 06e2162eea99c8e4c5cc6a9c0de1b3eaee92f982 Mon Sep 17 00:00:00 2001 From: Xiaohan-Tian <157918347+Xiaohan-Tian@users.noreply.github.com> Date: Tue, 19 May 2026 17:13:13 -0700 Subject: [PATCH] feat: added visual feedback when pressing piano keys --- src/components/piano-roll/PianoKeys.test.tsx | 146 +++++++++++++++++++ src/components/piano-roll/PianoKeys.tsx | 124 ++++++++++++---- src/components/piano-roll/PianoRoll.css | 30 ++++ src/core/midi-input/KGMidiInput.test.ts | 36 ++++- src/core/midi-input/KGMidiInput.ts | 29 ++++ 5 files changed, 331 insertions(+), 34 deletions(-) create mode 100644 src/components/piano-roll/PianoKeys.test.tsx diff --git a/src/components/piano-roll/PianoKeys.test.tsx b/src/components/piano-roll/PianoKeys.test.tsx new file mode 100644 index 0000000..bc86fb4 --- /dev/null +++ b/src/components/piano-roll/PianoKeys.test.tsx @@ -0,0 +1,146 @@ +import React from 'react'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { act, fireEvent, render, screen } from '@testing-library/react'; +import PianoKeys from './PianoKeys'; +import { createMockMidiNote, createMockMidiRegion, createMockMidiTrack } from '../../test/utils/mock-data'; + +type TestLiveNoteActivityListener = (...args: [{ pitch: number; isNoteOn: boolean }]) => void; + +const storeState = { + tracks: [createMockMidiTrack({ id: 1 })], + playheadPosition: 0, + isPlaying: false, +}; + +const audioInterfaceMock = { + getIsInitialized: vi.fn(), + getIsAudioContextStarted: vi.fn(), + startAudioContext: vi.fn(), + triggerNoteAttack: vi.fn(), + releaseNote: vi.fn(), +}; + +let liveNoteActivityListener: TestLiveNoteActivityListener | null = null; +const midiInputMock = { + addLiveNoteActivityListener: vi.fn((listener: TestLiveNoteActivityListener) => { + liveNoteActivityListener = listener; + }), + removeLiveNoteActivityListener: vi.fn((listener: TestLiveNoteActivityListener) => { + if (liveNoteActivityListener === listener) { + liveNoteActivityListener = null; + } + }), +}; + +vi.mock('../../stores/projectStore', () => ({ + useProjectStore: (selector: (...args: [typeof storeState]) => unknown) => selector(storeState), +})); + +vi.mock('../../core/audio-interface/KGAudioInterface', () => ({ + KGAudioInterface: { + instance: () => audioInterfaceMock, + }, +})); + +vi.mock('../../core/midi-input/KGMidiInput', () => ({ + KGMidiInput: { + instance: () => midiInputMock, + }, +})); + +describe('PianoKeys', () => { + const activeRegion = createMockMidiRegion({ + trackId: '1', + notes: [createMockMidiNote({ id: 'note-c4', pitch: 60, startBeat: 0, endBeat: 2 })], + }); + + beforeEach(() => { + storeState.tracks = [createMockMidiTrack({ id: 1 })]; + storeState.playheadPosition = 0; + storeState.isPlaying = false; + liveNoteActivityListener = null; + vi.clearAllMocks(); + audioInterfaceMock.getIsInitialized.mockReturnValue(true); + audioInterfaceMock.getIsAudioContextStarted.mockReturnValue(true); + audioInterfaceMock.startAudioContext.mockResolvedValue(undefined); + }); + + it('shows dot and background feedback for mouse preview while held', () => { + const { container } = render(); + const key = container.querySelector('[data-note="C4"]') as HTMLElement; + + fireEvent.mouseDown(key); + + expect(key.className).toContain('visual-active'); + expect(screen.getByTestId('piano-key-dot-C4')).toBeInTheDocument(); + + fireEvent.mouseUp(key); + + expect(key.className).not.toContain('visual-active'); + expect(screen.queryByTestId('piano-key-dot-C4')).not.toBeInTheDocument(); + }); + + it('shows MIDI activity dot without background feedback', () => { + const { container } = render(); + const key = container.querySelector('[data-note="C4"]') as HTMLElement; + expect(midiInputMock.addLiveNoteActivityListener).toHaveBeenCalledTimes(1); + + act(() => { + liveNoteActivityListener?.({ pitch: 60, isNoteOn: true }); + }); + + expect(screen.getByTestId('piano-key-dot-C4')).toBeInTheDocument(); + expect(key.className).not.toContain('visual-active'); + + act(() => { + liveNoteActivityListener?.({ pitch: 60, isNoteOn: false }); + }); + + expect(screen.queryByTestId('piano-key-dot-C4')).not.toBeInTheDocument(); + }); + + it('shows playback background feedback without dot for sounding notes in the active region', () => { + storeState.isPlaying = true; + storeState.playheadPosition = 1; + + const { container } = render(); + const key = container.querySelector('[data-note="C4"]') as HTMLElement; + + expect(key.className).toContain('playback-active'); + expect(key.className).toContain('visual-active'); + expect(screen.queryByTestId('piano-key-dot-C4')).not.toBeInTheDocument(); + }); + + it('preserves source-specific feedback while mouse, MIDI, and playback overlap', () => { + storeState.isPlaying = true; + storeState.playheadPosition = 1; + + const { container, rerender } = render(); + const key = container.querySelector('[data-note="C4"]') as HTMLElement; + + fireEvent.mouseDown(key); + act(() => { + liveNoteActivityListener?.({ pitch: 60, isNoteOn: true }); + }); + + expect(key.className).toContain('visual-active'); + expect(screen.getByTestId('piano-key-dot-C4')).toBeInTheDocument(); + + storeState.isPlaying = false; + rerender(); + + expect(key.className).toContain('visual-active'); + expect(screen.getByTestId('piano-key-dot-C4')).toBeInTheDocument(); + + fireEvent.mouseUp(key); + + expect(key.className).not.toContain('visual-active'); + expect(screen.getByTestId('piano-key-dot-C4')).toBeInTheDocument(); + + act(() => { + liveNoteActivityListener?.({ pitch: 60, isNoteOn: false }); + }); + + expect(screen.queryByTestId('piano-key-dot-C4')).not.toBeInTheDocument(); + }); +}); diff --git a/src/components/piano-roll/PianoKeys.tsx b/src/components/piano-roll/PianoKeys.tsx index e62519a..a4e8fe4 100644 --- a/src/components/piano-roll/PianoKeys.tsx +++ b/src/components/piano-roll/PianoKeys.tsx @@ -1,30 +1,94 @@ -import React, { useState, useRef } from 'react'; +import React, { useEffect, useMemo, useRef, useState } from 'react'; import { KGMidiRegion } from '../../core/region/KGMidiRegion'; import { KGAudioInterface } from '../../core/audio-interface/KGAudioInterface'; -import { noteNameToPitch, midiPercussionKeyMap, pitchToNoteNameString } from '../../util/midiUtil'; +import { noteNameToPitch, midiPercussionKeyMap } from '../../util/midiUtil'; import { useProjectStore } from '../../stores/projectStore'; import { KGMidiTrack } from '../../core/track/KGMidiTrack'; +import { KGMidiInput, type LiveMidiNoteActivityEvent } from '../../core/midi-input/KGMidiInput'; interface PianoKeysProps { activeRegion: KGMidiRegion | null; } +function incrementPitchCount(source: Map, pitch: number): Map { + const next = new Map(source); + next.set(pitch, (next.get(pitch) ?? 0) + 1); + return next; +} + +function decrementPitchCount(source: Map, pitch: number): Map { + const next = new Map(source); + const current = next.get(pitch) ?? 0; + + if (current <= 1) { + next.delete(pitch); + } else { + next.set(pitch, current - 1); + } + + return next; +} + const PianoKeys: React.FC = ({ activeRegion }) => { - const [pressedKeys, setPressedKeys] = useState>(new Set()); - const pressedKeysRef = useRef>(new Set()); - const { tracks } = useProjectStore(); + const [mouseActivePitches, setMouseActivePitches] = useState>(new Map()); + const [midiActivePitches, setMidiActivePitches] = useState>(new Map()); + const mouseActivePitchesRef = useRef>(new Map()); + const tracks = useProjectStore(state => state.tracks); + const playheadPosition = useProjectStore(state => state.playheadPosition); + const isPlaying = useProjectStore(state => state.isPlaying); // Check if current active region belongs to a drum track - const isDrumTrack = React.useMemo(() => { + const isDrumTrack = useMemo(() => { if (!activeRegion) return false; const track = tracks.find(t => t.getId().toString() === activeRegion.getTrackId()); return track instanceof KGMidiTrack && track.getInstrument() === 'standard'; }, [activeRegion, tracks]); + const playbackActivePitches = useMemo(() => { + if (!activeRegion || !isPlaying) { + return new Set(); + } + + const activePitches = new Set(); + const absolutePlayhead = playheadPosition; + const regionStartBeat = activeRegion.getStartFromBeat(); + + activeRegion.getNotes().forEach(note => { + const startBeat = regionStartBeat + note.getStartBeat(); + const endBeat = regionStartBeat + note.getEndBeat(); + + if (absolutePlayhead >= startBeat && absolutePlayhead < endBeat) { + activePitches.add(note.getPitch()); + } + }); + + return activePitches; + }, [activeRegion, isPlaying, playheadPosition]); + + useEffect(() => { + const midiInput = KGMidiInput.instance(); + + const handleLiveNoteActivity = (event: LiveMidiNoteActivityEvent) => { + setMidiActivePitches(current => ( + event.isNoteOn + ? incrementPitchCount(current, event.pitch) + : decrementPitchCount(current, event.pitch) + )); + }; + + midiInput.addLiveNoteActivityListener(handleLiveNoteActivity); + + return () => { + midiInput.removeLiveNoteActivityListener(handleLiveNoteActivity); + }; + }, []); + // Handle mouse down on piano key const handleKeyMouseDown = (keyId: string) => { + const pitch = noteNameToPitch(keyId); + // Prevent double pressing the same key - if (pressedKeysRef.current.has(keyId)) { + if ((mouseActivePitchesRef.current.get(pitch) ?? 0) > 0) { return; } @@ -37,9 +101,6 @@ const PianoKeys: React.FC = ({ activeRegion }) => { const trackId = activeRegion.getTrackId(); try { - // Convert note name to pitch (keyId is always a note name like "C4") - const pitch = noteNameToPitch(keyId); - // Get audio interface and start playing the note const audioInterface = KGAudioInterface.instance(); if (audioInterface.getIsInitialized()) { @@ -54,11 +115,9 @@ const PianoKeys: React.FC = ({ activeRegion }) => { if (audioInterface.getIsAudioContextStarted()) { audioInterface.triggerNoteAttack(trackId, pitch, 127); - // Update pressed keys state - const newPressedKeys = new Set(pressedKeysRef.current); - newPressedKeys.add(keyId); - pressedKeysRef.current = newPressedKeys; - setPressedKeys(newPressedKeys); + const nextMouseActivePitches = incrementPitchCount(mouseActivePitchesRef.current, pitch); + mouseActivePitchesRef.current = nextMouseActivePitches; + setMouseActivePitches(nextMouseActivePitches); console.log(`Started playing piano key: ${keyId} (pitch ${pitch})`); } @@ -70,8 +129,10 @@ const PianoKeys: React.FC = ({ activeRegion }) => { // Handle mouse up on piano key const handleKeyMouseUp = (keyId: string) => { + const pitch = noteNameToPitch(keyId); + // Only release if key was actually pressed - if (!pressedKeysRef.current.has(keyId)) { + if ((mouseActivePitchesRef.current.get(pitch) ?? 0) === 0) { return; } @@ -83,19 +144,14 @@ const PianoKeys: React.FC = ({ activeRegion }) => { const trackId = activeRegion.getTrackId(); try { - // Convert note name to pitch (keyId is always a note name like "C4") - const pitch = noteNameToPitch(keyId); - // Get audio interface and stop playing the note const audioInterface = KGAudioInterface.instance(); if (audioInterface.getIsInitialized() && audioInterface.getIsAudioContextStarted()) { audioInterface.releaseNote(trackId, pitch); - // Update pressed keys state - const newPressedKeys = new Set(pressedKeysRef.current); - newPressedKeys.delete(keyId); - pressedKeysRef.current = newPressedKeys; - setPressedKeys(newPressedKeys); + const nextMouseActivePitches = decrementPitchCount(mouseActivePitchesRef.current, pitch); + mouseActivePitchesRef.current = nextMouseActivePitches; + setMouseActivePitches(nextMouseActivePitches); console.log(`Stopped playing piano key: ${keyId} (pitch ${pitch})`); } @@ -123,14 +179,25 @@ const PianoKeys: React.FC = ({ activeRegion }) => { const note = notes[i]; const isSharp = note.includes('#'); const keyId = `${note}${octave}`; - const isPressed = pressedKeys.has(keyId); - const keyClass = `piano-key ${isSharp ? 'sharp' : 'natural'} ${isPressed ? 'pressed' : ''}`; + const pitch = noteNameToPitch(keyId); + const isMouseActive = (mouseActivePitches.get(pitch) ?? 0) > 0; + const isMidiActive = (midiActivePitches.get(pitch) ?? 0) > 0; + const isPlaybackActive = playbackActivePitches.has(pitch); + const showIndicator = isMouseActive || isMidiActive; + const hasBackgroundFeedback = isMouseActive || isPlaybackActive; + const keyClass = [ + 'piano-key', + isSharp ? 'sharp' : 'natural', + isMouseActive ? 'mouse-active' : '', + isMidiActive ? 'midi-active' : '', + isPlaybackActive ? 'playback-active' : '', + hasBackgroundFeedback ? 'visual-active' : '', + ].filter(Boolean).join(' '); const isC = note === 'C'; // For drum tracks, show drum labels when available let labelContent = null; if (isDrumTrack) { - const pitch = noteNameToPitch(keyId); const drumInfo = midiPercussionKeyMap[pitch]; if (drumInfo) { labelContent = {drumInfo.shortName}; @@ -153,6 +220,7 @@ const PianoKeys: React.FC = ({ activeRegion }) => { }} > {labelContent} + {showIndicator ? : null} ); } @@ -175,4 +243,4 @@ const PianoKeys: React.FC = ({ activeRegion }) => { ); }; -export default PianoKeys; \ No newline at end of file +export default PianoKeys; diff --git a/src/components/piano-roll/PianoRoll.css b/src/components/piano-roll/PianoRoll.css index 977042a..794960d 100644 --- a/src/components/piano-roll/PianoRoll.css +++ b/src/components/piano-roll/PianoRoll.css @@ -572,6 +572,8 @@ display: flex; align-items: center; border-bottom: 1px solid #3a3a3a; + position: relative; + transition: background-color 0.08s ease; } .piano-key.natural { @@ -584,9 +586,37 @@ color: #e0e0e0; } +.piano-key.natural.visual-active { + background-color: #b8b8b8; +} + +.piano-key.sharp.visual-active { + background-color: #5a5a5a; +} + .key-label { font-size: 10px; padding-left: 5px; + padding-right: 18px; + position: relative; + z-index: 1; +} + +.piano-key-activity-dot { + position: absolute; + right: 6px; + top: 50%; + width: 8px; + height: 8px; + margin-top: -4px; + border-radius: 999px; + background-color: #000; + border: 1px solid #000; +} + +.piano-key.sharp .piano-key-activity-dot { + background-color: #fff; + border-color: #fff; } .piano-grid { diff --git a/src/core/midi-input/KGMidiInput.test.ts b/src/core/midi-input/KGMidiInput.test.ts index 9e53747..396f04c 100644 --- a/src/core/midi-input/KGMidiInput.test.ts +++ b/src/core/midi-input/KGMidiInput.test.ts @@ -2,6 +2,9 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; import { KGAudioTrack } from '../track/KGAudioTrack'; import { KGMidiTrack } from '../track/KGMidiTrack'; +type TestMidiEvent = { data: Uint8Array }; +type TestLiveNoteActivityListener = (...args: [{ pitch: number; isNoteOn: boolean }]) => void; + const { getStateMock, audioInterfaceMock } = vi.hoisted(() => ({ getStateMock: vi.fn(), audioInterfaceMock: { @@ -45,19 +48,40 @@ describe('KGMidiInput pitch bend', () => { it('routes live MIDI note on/off through the live monitoring path', () => { const midiInput = KGMidiInput.instance() as unknown as { - handleMIDIMessage: (event: { data: Uint8Array }) => void; + handleMIDIMessage: (...args: [TestMidiEvent]) => void; + addLiveNoteActivityListener: (...args: [TestLiveNoteActivityListener]) => void; }; + const listener = vi.fn(); + + midiInput.addLiveNoteActivityListener(listener); midiInput.handleMIDIMessage({ data: new Uint8Array([0x90, 60, 100]) }); midiInput.handleMIDIMessage({ data: new Uint8Array([0x80, 60, 0]) }); expect(audioInterfaceMock.triggerLiveMidiNoteAttack).toHaveBeenCalledWith('1', 60, 100); expect(audioInterfaceMock.releaseLiveMidiNote).toHaveBeenCalledWith('1', 60); + expect(listener).toHaveBeenNthCalledWith(1, { pitch: 60, isNoteOn: true }); + expect(listener).toHaveBeenNthCalledWith(2, { pitch: 60, isNoteOn: false }); + }); + + it('stops notifying removed live note activity listeners', () => { + const midiInput = KGMidiInput.instance() as unknown as { + handleMIDIMessage: (...args: [TestMidiEvent]) => void; + addLiveNoteActivityListener: (...args: [TestLiveNoteActivityListener]) => void; + removeLiveNoteActivityListener: (...args: [TestLiveNoteActivityListener]) => void; + }; + const listener = vi.fn(); + + midiInput.addLiveNoteActivityListener(listener); + midiInput.removeLiveNoteActivityListener(listener); + midiInput.handleMIDIMessage({ data: new Uint8Array([0x90, 60, 100]) }); + + expect(listener).not.toHaveBeenCalled(); }); it('latches live note ownership to the note-on track', () => { const midiInput = KGMidiInput.instance() as unknown as { - handleMIDIMessage: (event: { data: Uint8Array }) => void; + handleMIDIMessage: (...args: [TestMidiEvent]) => void; }; midiInput.handleMIDIMessage({ data: new Uint8Array([0x90, 60, 100]) }); @@ -73,7 +97,7 @@ describe('KGMidiInput pitch bend', () => { it('normalizes MIDI pitch bend and forwards it to the selected track', () => { const midiInput = KGMidiInput.instance() as unknown as { - handleMIDIMessage: (event: { data: Uint8Array }) => void; + handleMIDIMessage: (...args: [TestMidiEvent]) => void; }; midiInput.handleMIDIMessage({ data: new Uint8Array([0xe0, 0x00, 0x40]) }); @@ -87,7 +111,7 @@ describe('KGMidiInput pitch bend', () => { it('maps supported CC messages to live expression and sustain for standard pedals', () => { const midiInput = KGMidiInput.instance() as unknown as { - handleMIDIMessage: (event: { data: Uint8Array }) => void; + handleMIDIMessage: (...args: [TestMidiEvent]) => void; }; midiInput.handleMIDIMessage({ data: new Uint8Array([0xb0, 0x01, 0x20]) }); @@ -106,7 +130,7 @@ describe('KGMidiInput pitch bend', () => { it('calibrates inverted sustain pedals from the first observed CC64 message', () => { const midiInput = KGMidiInput.instance() as unknown as { - handleMIDIMessage: (event: { data: Uint8Array }) => void; + handleMIDIMessage: (...args: [TestMidiEvent]) => void; }; midiInput.handleMIDIMessage({ data: new Uint8Array([0xb0, 0x40, 0x00]) }); @@ -123,7 +147,7 @@ describe('KGMidiInput pitch bend', () => { }); const midiInput = KGMidiInput.instance() as unknown as { - handleMIDIMessage: (event: { data: Uint8Array }) => void; + handleMIDIMessage: (...args: [TestMidiEvent]) => void; }; midiInput.handleMIDIMessage({ data: new Uint8Array([0x90, 60, 100]) }); diff --git a/src/core/midi-input/KGMidiInput.ts b/src/core/midi-input/KGMidiInput.ts index 1ee0213..f85aa11 100644 --- a/src/core/midi-input/KGMidiInput.ts +++ b/src/core/midi-input/KGMidiInput.ts @@ -2,6 +2,13 @@ import { KGAudioInterface } from '../audio-interface/KGAudioInterface'; import { useProjectStore } from '../../stores/projectStore'; import { KGMidiTrack } from '../track/KGMidiTrack'; +export interface LiveMidiNoteActivityEvent { + pitch: number; + isNoteOn: boolean; +} + +type LiveNoteActivityListener = (...args: [LiveMidiNoteActivityEvent]) => void; + /** * KGMidiInput - MIDI input manager for the DAW * Implements the singleton pattern for global MIDI device management @@ -32,6 +39,7 @@ export class KGMidiInput { private onRecordControlChange: ((controller: number, value: number) => void) | null = null; private liveNoteTrackOwnership: Map = new Map(); private sustainPolarityInverted: boolean | null = null; + private liveNoteActivityListeners: LiveNoteActivityListener[] = []; // Private constructor to prevent direct instantiation private constructor() { @@ -178,12 +186,14 @@ export class KGMidiInput { // Note On: command = 0x90 (144) if (command === 0x90 && velocity > 0) { console.log(`MIDI Note On: pitch=${pitch}, velocity=${velocity}, channel=${channel}`); + this.emitLiveNoteActivity({ pitch, isNoteOn: true }); this.triggerNoteOn(pitch, velocity); this.onRecordNoteOn?.(pitch, velocity); } // Note Off: command = 0x80 (128) or Note On with velocity 0 else if (command === 0x80 || (command === 0x90 && velocity === 0)) { console.log(`MIDI Note Off: pitch=${pitch}, channel=${channel}`); + this.emitLiveNoteActivity({ pitch, isNoteOn: false }); this.triggerNoteOff(pitch); this.onRecordNoteOff?.(pitch); } @@ -354,6 +364,16 @@ export class KGMidiInput { return this.sustainPolarityInverted ? !rawPressed : rawPressed; } + private emitLiveNoteActivity(event: LiveMidiNoteActivityEvent): void { + for (const listener of this.liveNoteActivityListeners) { + try { + listener(event); + } catch { + // Swallow listener errors to avoid disrupting MIDI handling. + } + } + } + /** * Clean up MIDI resources */ @@ -374,6 +394,7 @@ export class KGMidiInput { this.isInitialized = false; this.liveNoteTrackOwnership.clear(); this.sustainPolarityInverted = null; + this.liveNoteActivityListeners = []; console.log("MIDI resources disposed successfully"); } catch (error) { @@ -395,6 +416,14 @@ export class KGMidiInput { this.onRecordControlChange = onControlChange; } + public addLiveNoteActivityListener(listener: LiveNoteActivityListener): void { + this.liveNoteActivityListeners.push(listener); + } + + public removeLiveNoteActivityListener(listener: LiveNoteActivityListener): void { + this.liveNoteActivityListeners = this.liveNoteActivityListeners.filter(current => current !== listener); + } + // ===== GETTERS ===== public getIsInitialized(): boolean {