From 00c783eea9fabdee4543974c45338ae02a38a292 Mon Sep 17 00:00:00 2001 From: riversedge Date: Thu, 5 Feb 2026 12:09:48 -0500 Subject: [PATCH] Some drag and drop and other UI enhancements --- App.tsx | 21 ++ components/CreatePanel.tsx | 363 ++++++++++++++++++++++------ components/SongDropdownMenu.tsx | 18 +- components/SongList.tsx | 189 ++++++++++++--- server/src/routes/referenceTrack.ts | 136 ++++++++++- 5 files changed, 623 insertions(+), 104 deletions(-) diff --git a/App.tsx b/App.tsx index 0a7e37f..3f00307 100644 --- a/App.tsx +++ b/App.tsx @@ -66,6 +66,7 @@ export default function App() { // UI State const [isGenerating, setIsGenerating] = useState(false); const [showRightSidebar, setShowRightSidebar] = useState(true); + const [pendingAudioSelection, setPendingAudioSelection] = useState<{ target: 'reference' | 'source'; url: string; title?: string } | null>(null); // Mobile UI Toggle const [mobileShowList, setMobileShowList] = useState(false); @@ -1066,6 +1067,20 @@ export default function App() { window.history.pushState({}, '', `/playlist/${playlistId}`); }; + const handleUseAsReference = (song: Song) => { + if (!song.audioUrl) return; + setPendingAudioSelection({ target: 'reference', url: song.audioUrl, title: song.title }); + setCurrentView('create'); + setMobileShowList(false); + }; + + const handleCoverSong = (song: Song) => { + if (!song.audioUrl) return; + setPendingAudioSelection({ target: 'source', url: song.audioUrl, title: song.title }); + setCurrentView('create'); + setMobileShowList(false); + }; + const handleBackFromPlaylist = () => { setViewingPlaylistId(null); setCurrentView('library'); @@ -1184,6 +1199,9 @@ export default function App() { onGenerate={handleGenerate} isGenerating={isGenerating} initialData={reuseData} + createdSongs={songs} + pendingAudioSelection={pendingAudioSelection} + onAudioSelectionApplied={() => setPendingAudioSelection(null)} /> @@ -1198,6 +1216,7 @@ export default function App() { selectedSong={selectedSong} likedSongIds={likedSongIds} isPlaying={isPlaying} + referenceTracks={referenceTracks} onPlay={playSong} onSelect={(s) => { setSelectedSong(s); @@ -1211,6 +1230,8 @@ export default function App() { onReusePrompt={handleReuse} onDelete={handleDeleteSong} onDeleteMany={handleDeleteSongs} + onUseAsReference={handleUseAsReference} + onCoverSong={handleCoverSong} /> diff --git a/components/CreatePanel.tsx b/components/CreatePanel.tsx index e84758c..268f880 100644 --- a/components/CreatePanel.tsx +++ b/components/CreatePanel.tsx @@ -1,4 +1,4 @@ -import React, { useState, useEffect, useRef, useCallback } from 'react'; +import React, { useState, useEffect, useRef, useCallback, useMemo } from 'react'; import { Sparkles, ChevronDown, Settings2, Trash2, Music2, Sliders, Dices, Hash, RefreshCw, Plus, Upload, Play, Pause, Loader2 } from 'lucide-react'; import { GenerationParams, Song } from '../types'; import { useAuth } from '../context/AuthContext'; @@ -19,6 +19,9 @@ interface CreatePanelProps { onGenerate: (params: GenerationParams) => void; isGenerating: boolean; initialData?: { song: Song, timestamp: number } | null; + createdSongs?: Song[]; + pendingAudioSelection?: { target: 'reference' | 'source'; url: string; title?: string } | null; + onAudioSelectionApplied?: () => void; } const KEY_SIGNATURES = [ @@ -98,8 +101,15 @@ const VOCAL_LANGUAGES = [ { value: 'zh', label: 'Chinese (Mandarin)' }, ]; -export const CreatePanel: React.FC = ({ onGenerate, isGenerating, initialData }) => { - const { isAuthenticated, token } = useAuth(); +export const CreatePanel: React.FC = ({ + onGenerate, + isGenerating, + initialData, + createdSongs = [], + pendingAudioSelection, + onAudioSelectionApplied, +}) => { + const { isAuthenticated, token, user } = useAuth(); // Mode const [customMode, setCustomMode] = useState(true); @@ -177,10 +187,13 @@ export const CreatePanel: React.FC = ({ onGenerate, isGenerati const [isUploadingReference, setIsUploadingReference] = useState(false); const [isUploadingSource, setIsUploadingSource] = useState(false); + const [isTranscribingReference, setIsTranscribingReference] = useState(false); + const transcribeAbortRef = useRef(null); const [uploadError, setUploadError] = useState(null); const [isFormattingStyle, setIsFormattingStyle] = useState(false); const [isFormattingLyrics, setIsFormattingLyrics] = useState(false); const [isDraggingFile, setIsDraggingFile] = useState(false); + const [dragKind, setDragKind] = useState<'file' | 'audio' | null>(null); const referenceInputRef = useRef(null); const sourceInputRef = useRef(null); const dragDepthRef = useRef(0); @@ -201,9 +214,24 @@ export const CreatePanel: React.FC = ({ onGenerate, isGenerati const [referenceTracks, setReferenceTracks] = useState([]); const [isLoadingTracks, setIsLoadingTracks] = useState(false); const [playingTrackId, setPlayingTrackId] = useState(null); + const [playingTrackSource, setPlayingTrackSource] = useState<'uploads' | 'created' | null>(null); const modalAudioRef = useRef(null); const [modalTrackTime, setModalTrackTime] = useState(0); const [modalTrackDuration, setModalTrackDuration] = useState(0); + const [libraryTab, setLibraryTab] = useState<'uploads' | 'created'>('uploads'); + + const createdTrackOptions = useMemo(() => { + return createdSongs + .filter(song => !song.isGenerating) + .filter(song => (user ? song.userId === user.id : true)) + .filter(song => Boolean(song.audioUrl)) + .map(song => ({ + id: song.id, + title: song.title || 'Untitled', + audio_url: song.audioUrl!, + duration: song.duration, + })); + }, [createdSongs, user]); const getAudioLabel = (url: string) => { try { @@ -236,6 +264,16 @@ export const CreatePanel: React.FC = ({ onGenerate, isGenerati } }, [initialData]); + useEffect(() => { + if (!pendingAudioSelection) return; + applyAudioTargetUrl( + pendingAudioSelection.target, + pendingAudioSelection.url, + pendingAudioSelection.title + ); + onAudioSelectionApplied?.(); + }, [pendingAudioSelection, onAudioSelectionApplied]); + useEffect(() => { const handleMouseMove = (e: MouseEvent) => { if (!isResizing) return; @@ -312,34 +350,47 @@ export const CreatePanel: React.FC = ({ onGenerate, isGenerati }, [duration, activeMaxDuration]); useEffect(() => { - const isFileDrag = (e: DragEvent) => - !!(e.dataTransfer && Array.from(e.dataTransfer.types).includes('Files')); + const getDragKind = (e: DragEvent): 'file' | 'audio' | null => { + if (!e.dataTransfer) return null; + const types = Array.from(e.dataTransfer.types); + if (types.includes('Files')) return 'file'; + if (types.includes('application/x-ace-audio')) return 'audio'; + return null; + }; const handleDragEnter = (e: DragEvent) => { - if (!isFileDrag(e)) return; + const kind = getDragKind(e); + if (!kind) return; dragDepthRef.current += 1; setIsDraggingFile(true); + setDragKind(kind); e.preventDefault(); }; const handleDragOver = (e: DragEvent) => { - if (!isFileDrag(e)) return; + const kind = getDragKind(e); + if (!kind) return; + setDragKind(kind); e.preventDefault(); }; const handleDragLeave = (e: DragEvent) => { - if (!isFileDrag(e)) return; + const kind = getDragKind(e); + if (!kind) return; dragDepthRef.current = Math.max(0, dragDepthRef.current - 1); if (dragDepthRef.current === 0) { setIsDraggingFile(false); + setDragKind(null); } }; const handleDrop = (e: DragEvent) => { - if (!isFileDrag(e)) return; + const kind = getDragKind(e); + if (!kind) return; e.preventDefault(); dragDepthRef.current = 0; setIsDraggingFile(false); + setDragKind(null); }; window.addEventListener('dragenter', handleDragEnter); @@ -437,9 +488,10 @@ export const CreatePanel: React.FC = ({ onGenerate, isGenerati } }; - const openAudioModal = (target: 'reference' | 'source') => { + const openAudioModal = (target: 'reference' | 'source', tab: 'uploads' | 'created' = 'uploads') => { setAudioModalTarget(target); setTempAudioUrl(''); + setLibraryTab(tab); setShowAudioModal(true); void fetchReferenceTracks(); }; @@ -490,7 +542,11 @@ export const CreatePanel: React.FC = ({ onGenerate, isGenerati // Also set as current reference/source const selectedTarget = target ?? audioModalTarget; applyAudioTargetUrl(selectedTarget, data.track.audio_url, data.track.filename); - setShowAudioModal(false); + if (data.whisper_available && data.track?.id) { + void transcribeReferenceTrack(data.track.id).then(() => undefined); + } else { + setShowAudioModal(false); + } } catch (err) { const message = err instanceof Error ? err.message : 'Upload failed'; setUploadError(message); @@ -499,6 +555,43 @@ export const CreatePanel: React.FC = ({ onGenerate, isGenerati } }; + const transcribeReferenceTrack = async (trackId: string) => { + if (!token) return; + setIsTranscribingReference(true); + const controller = new AbortController(); + transcribeAbortRef.current = controller; + try { + const response = await fetch(`/api/reference-tracks/${trackId}/transcribe`, { + method: 'POST', + headers: { Authorization: `Bearer ${token}` }, + signal: controller.signal, + }); + if (!response.ok) { + throw new Error('Failed to transcribe'); + } + const data = await response.json(); + if (data.lyrics) { + setLyrics(prev => prev || data.lyrics); + } + } catch (err) { + if (controller.signal.aborted) return; + console.error('Transcription failed:', err); + } finally { + if (transcribeAbortRef.current === controller) { + transcribeAbortRef.current = null; + } + setIsTranscribingReference(false); + } + }; + + const cancelTranscription = () => { + if (transcribeAbortRef.current) { + transcribeAbortRef.current.abort(); + transcribeAbortRef.current = null; + } + setIsTranscribingReference(false); + }; + const deleteReferenceTrack = async (trackId: string) => { if (!token) return; try { @@ -508,8 +601,9 @@ export const CreatePanel: React.FC = ({ onGenerate, isGenerati }); if (response.ok) { setReferenceTracks(prev => prev.filter(t => t.id !== trackId)); - if (playingTrackId === trackId) { + if (playingTrackId === trackId && playingTrackSource === 'uploads') { setPlayingTrackId(null); + setPlayingTrackSource(null); if (modalAudioRef.current) { modalAudioRef.current.pause(); } @@ -520,20 +614,23 @@ export const CreatePanel: React.FC = ({ onGenerate, isGenerati } }; - const useReferenceTrack = (track: ReferenceTrack) => { - applyAudioTargetUrl(audioModalTarget, track.audio_url, track.filename); + const useReferenceTrack = (track: { audio_url: string; title?: string }) => { + applyAudioTargetUrl(audioModalTarget, track.audio_url, track.title); setShowAudioModal(false); setPlayingTrackId(null); + setPlayingTrackSource(null); }; - const toggleModalTrack = (track: ReferenceTrack) => { + const toggleModalTrack = (track: { id: string; audio_url: string; source: 'uploads' | 'created' }) => { if (playingTrackId === track.id) { if (modalAudioRef.current) { modalAudioRef.current.pause(); } setPlayingTrackId(null); + setPlayingTrackSource(null); } else { setPlayingTrackId(track.id); + setPlayingTrackSource(track.source); if (modalAudioRef.current) { modalAudioRef.current.src = track.audio_url; modalAudioRef.current.play().catch(() => undefined); @@ -588,6 +685,18 @@ export const CreatePanel: React.FC = ({ onGenerate, isGenerati const file = e.dataTransfer.files?.[0]; if (file) { void uploadReferenceTrack(file, target); + return; + } + const payload = e.dataTransfer.getData('application/x-ace-audio'); + if (payload) { + try { + const data = JSON.parse(payload); + if (data?.url) { + applyAudioTargetUrl(target, data.url, data.title); + } + } catch { + // ignore + } } }; @@ -602,7 +711,7 @@ export const CreatePanel: React.FC = ({ onGenerate, isGenerati }; const handleWorkspaceDragOver = (e: React.DragEvent) => { - if (e.dataTransfer.types.includes('Files')) { + if (e.dataTransfer.types.includes('Files') || e.dataTransfer.types.includes('application/x-ace-audio')) { e.preventDefault(); } }; @@ -706,12 +815,16 @@ export const CreatePanel: React.FC = ({ onGenerate, isGenerati
-
- +
+ {dragKind === 'audio' ? : } +
+
+ {dragKind === 'audio' ? 'Drop to use audio' : 'Drop to upload'}
-
Drop to upload
- Uploading as {audioTab === 'reference' ? 'Reference' : 'Cover'} + {dragKind === 'audio' + ? `Using as ${audioTab === 'reference' ? 'Reference' : 'Cover'}` + : `Uploading as ${audioTab === 'reference' ? 'Reference' : 'Cover'}`}
@@ -1072,13 +1185,13 @@ export const CreatePanel: React.FC = ({ onGenerate, isGenerati
+
+ )}
- {/* Mine Section */} + {/* Library Section */}
- - Mine - +
+ + +
{/* Track List */}
- {isLoadingTracks ? ( -
- -

Loading tracks...

-
- ) : referenceTracks.length === 0 ? ( + {libraryTab === 'uploads' ? ( + isLoadingTracks ? ( +
+ +

Loading tracks...

+
+ ) : referenceTracks.length === 0 ? ( +
+ +

No uploads yet

+

Upload audio files to use them as references

+
+ ) : ( +
+ {referenceTracks.map((track) => ( +
+ {/* Play Button */} + + + {/* Track Info */} +
+
+ + {track.filename.replace(/\.[^/.]+$/, '')} + + {track.tags && track.tags.length > 0 && ( +
+ {track.tags.slice(0, 2).map((tag, i) => ( + + {tag} + + ))} +
+ )} +
+ {/* Progress bar with seek - show when this track is playing */} + {playingTrackId === track.id && playingTrackSource === 'uploads' ? ( +
+ + {formatTime(modalTrackTime)} + +
{ + if (modalAudioRef.current && modalTrackDuration > 0) { + const rect = e.currentTarget.getBoundingClientRect(); + const percent = (e.clientX - rect.left) / rect.width; + modalAudioRef.current.currentTime = percent * modalTrackDuration; + } + }} + > +
0 ? `${(modalTrackTime / modalTrackDuration) * 100}%` : '0%' }} + > +
+
+
+ + {formatTime(modalTrackDuration)} + +
+ ) : ( +
+ {track.duration ? formatTime(track.duration) : '--:--'} +
+ )} +
+ + {/* Actions */} +
+ + +
+
+ ))} +
+ ) + ) : createdTrackOptions.length === 0 ? (
-

No tracks yet

-

Upload audio files to use them as references

+

No created songs yet

+

Generate songs to reuse them as cover or reference

) : (
- {referenceTracks.map((track) => ( + {createdTrackOptions.map((track) => (
- {/* Play Button */} - {/* Track Info */}
-
- - {track.filename.replace(/\.[^/.]+$/, '')} - - {track.tags && track.tags.length > 0 && ( -
- {track.tags.slice(0, 2).map((tag, i) => ( - - {tag} - - ))} -
- )} +
+ {track.title}
- {/* Progress bar with seek - show when this track is playing */} - {playingTrackId === track.id ? ( + {playingTrackId === track.id && playingTrackSource === 'created' ? (
{formatTime(modalTrackTime)} @@ -2001,27 +2236,19 @@ export const CreatePanel: React.FC = ({ onGenerate, isGenerati
) : (
- {track.duration ? formatTime(track.duration) : '--:--'} + {track.duration || '--:--'}
)}
- {/* Actions */}
-
))} @@ -2043,7 +2270,7 @@ export const CreatePanel: React.FC = ({ onGenerate, isGenerati setModalTrackDuration(modalAudioRef.current.duration); // Update track duration in database if not set const track = referenceTracks.find(t => t.id === playingTrackId); - if (track && !track.duration && token) { + if (playingTrackSource === 'uploads' && track && !track.duration && token) { fetch(`/api/reference-tracks/${track.id}`, { method: 'PATCH', headers: { diff --git a/components/SongDropdownMenu.tsx b/components/SongDropdownMenu.tsx index d012cae..efb5378 100644 --- a/components/SongDropdownMenu.tsx +++ b/components/SongDropdownMenu.tsx @@ -26,6 +26,8 @@ interface SongDropdownMenuProps { onDownload?: () => void; onShare?: () => void; onDelete?: () => void; + onUseAsReference?: () => void; + onCoverSong?: () => void; } interface MenuItemProps { @@ -70,7 +72,9 @@ export const SongDropdownMenu: React.FC = ({ onAddToPlaylist, onDownload, onShare, - onDelete + onDelete, + onUseAsReference, + onCoverSong }) => { const menuRef = useRef(null); @@ -190,6 +194,18 @@ export const SongDropdownMenu: React.FC = ({ label="Reuse Prompt" onClick={() => handleAction(onReusePrompt)} /> + } + label="Use as Reference" + onClick={() => handleAction(onUseAsReference)} + disabled={!song.audioUrl} + /> + } + label="Cover Song" + onClick={() => handleAction(onCoverSong)} + disabled={!song.audioUrl} + /> diff --git a/components/SongList.tsx b/components/SongList.tsx index 70515ae..b126869 100644 --- a/components/SongList.tsx +++ b/components/SongList.tsx @@ -12,6 +12,7 @@ interface SongListProps { selectedSong: Song | null; likedSongIds: Set; isPlaying: boolean; + referenceTracks?: { id: string; filename: string; audio_url: string; duration?: number | null; created_at?: string }[]; onPlay: (song: Song) => void; onSelect: (song: Song) => void; onToggleLike: (songId: string) => void; @@ -22,6 +23,8 @@ interface SongListProps { onReusePrompt?: (song: Song) => void; onDelete?: (song: Song) => void; onDeleteMany?: (songs: Song[]) => void; + onUseAsReference?: (song: Song) => void; + onCoverSong?: (song: Song) => void; } // ... existing code ... @@ -44,6 +47,7 @@ export const SongList: React.FC = ({ selectedSong, likedSongIds, isPlaying, + referenceTracks = [], onPlay, onSelect, onToggleLike, @@ -53,7 +57,9 @@ export const SongList: React.FC = ({ onNavigateToProfile, onReusePrompt, onDelete, - onDeleteMany + onDeleteMany, + onUseAsReference, + onCoverSong }) => { const { user } = useAuth(); const [searchQuery, setSearchQuery] = useState(''); @@ -120,6 +126,31 @@ export const SongList: React.FC = ({ }); }, [songs, searchQuery, activeFilters, likedSongIds]); + const filteredUploads = useMemo(() => { + if (activeFilters.size > 0) return []; + if (!referenceTracks.length) return []; + return referenceTracks.filter(track => { + const title = track.filename.replace(/\.[^/.]+$/, ''); + return title.toLowerCase().includes(searchQuery.toLowerCase()); + }); + }, [referenceTracks, searchQuery, activeFilters]); + + const listItems = useMemo(() => { + const songItems = filteredSongs.map(song => ({ + type: 'song' as const, + id: song.id, + createdAt: song.createdAt, + song + })); + const uploadItems = filteredUploads.map(track => ({ + type: 'upload' as const, + id: track.id, + createdAt: new Date(track.created_at || Date.now()), + track + })); + return [...songItems, ...uploadItems].sort((a, b) => b.createdAt.getTime() - a.createdAt.getTime()); + }, [filteredSongs, filteredUploads]); + const selectableSongs = useMemo( () => filteredSongs.filter(song => !song.isGenerating), [filteredSongs] @@ -254,7 +285,7 @@ export const SongList: React.FC = ({ {/* List */}
{/* Reduced vertical spacing */} - {filteredSongs.length === 0 ? ( + {listItems.length === 0 ? (
@@ -268,36 +299,59 @@ export const SongList: React.FC = ({
) : ( - filteredSongs.map((song) => ( - onPlay(song)} - onSelect={() => onSelect(song)} - onToggleSelect={() => { - if (song.isGenerating) return; - setSelectedIds(prev => { - const next = new Set(prev); - if (next.has(song.id)) next.delete(song.id); - else next.add(song.id); - return next; - }); - }} - onToggleLike={() => onToggleLike(song.id)} - onAddToPlaylist={() => onAddToPlaylist(song)} - onOpenVideo={() => onOpenVideo && onOpenVideo(song)} - onShowDetails={() => onShowDetails && onShowDetails(song)} - onNavigateToProfile={onNavigateToProfile} - onReusePrompt={() => onReusePrompt?.(song)} - onDelete={() => onDelete?.(song)} - /> + listItems.map((item) => ( + item.type === 'song' ? ( + onPlay(item.song)} + onSelect={() => onSelect(item.song)} + onToggleSelect={() => { + if (item.song.isGenerating) return; + setSelectedIds(prev => { + const next = new Set(prev); + if (next.has(item.song.id)) next.delete(item.song.id); + else next.add(item.song.id); + return next; + }); + }} + onToggleLike={() => onToggleLike(item.song.id)} + onAddToPlaylist={() => onAddToPlaylist(item.song)} + onOpenVideo={() => onOpenVideo && onOpenVideo(item.song)} + onShowDetails={() => onShowDetails && onShowDetails(item.song)} + onNavigateToProfile={onNavigateToProfile} + onReusePrompt={() => onReusePrompt?.(item.song)} + onDelete={() => onDelete?.(item.song)} + onUseAsReference={() => onUseAsReference?.(item.song)} + onCoverSong={() => onCoverSong?.(item.song)} + /> + ) : ( + { + onPlay({ + id: `upload_${item.id}`, + title, + lyrics: '', + style: 'Upload', + coverUrl: '', + duration: '0:00', + createdAt: item.createdAt, + tags: [], + audioUrl, + isPublic: false, + } as Song); + }} + /> + ) )) )}
@@ -325,6 +379,8 @@ interface SongItemProps { onNavigateToProfile?: (username: string) => void; onReusePrompt?: () => void; onDelete?: () => void; + onUseAsReference?: () => void; + onCoverSong?: () => void; } const SongItem: React.FC = ({ @@ -345,7 +401,9 @@ const SongItem: React.FC = ({ onShowDetails, onNavigateToProfile, onReusePrompt, - onDelete + onDelete, + onUseAsReference, + onCoverSong }) => { const [showDropdown, setShowDropdown] = useState(false); const [shareModalOpen, setShareModalOpen] = useState(false); @@ -355,7 +413,17 @@ const SongItem: React.FC = ({ <>
{ + if (!song.audioUrl || song.isGenerating) return; + e.dataTransfer.effectAllowed = 'copy'; + e.dataTransfer.setData('application/x-ace-audio', JSON.stringify({ + url: song.audioUrl, + title: song.title || 'Untitled', + source: 'song', + })); + }} + className={`group flex items-center gap-4 p-2 rounded-lg hover:bg-zinc-100 dark:hover:bg-[#18181b] transition-all cursor-pointer border ${isSelected ? 'bg-zinc-100 dark:bg-[#18181b] border-zinc-200 dark:border-white/10' : 'border-transparent bg-transparent'} ${song.audioUrl && !song.isGenerating ? 'cursor-grab active:cursor-grabbing' : ''}`} > {isSelectionMode && (
@@ -577,3 +647,54 @@ const SongItem: React.FC = ({ ); }; + +const UploadItem: React.FC<{ + track: { id: string; filename: string; audio_url: string; duration?: number | null }; + onPlay: (audioUrl: string, title: string) => void; +}> = ({ track, onPlay }) => { + const title = track.filename.replace(/\.[^/.]+$/, ''); + const duration = track.duration + ? `${Math.floor(track.duration / 60)}:${String(Math.floor(track.duration % 60)).padStart(2, '0')}` + : '--:--'; + + return ( +
{ + e.dataTransfer.effectAllowed = 'copy'; + e.dataTransfer.setData('application/x-ace-audio', JSON.stringify({ + url: track.audio_url, + title, + source: 'upload', + })); + }} + > +
+ +
{ + e.stopPropagation(); + onPlay(track.audio_url, title); + }} + > +
+ +
+
+
+ +
+
+ {title} +
+
Upload
+
+ +
+ {duration} +
+
+ ); +}; diff --git a/server/src/routes/referenceTrack.ts b/server/src/routes/referenceTrack.ts index 208b639..4510b2f 100644 --- a/server/src/routes/referenceTrack.ts +++ b/server/src/routes/referenceTrack.ts @@ -1,11 +1,18 @@ import { Router, Response } from 'express'; import multer from 'multer'; import path from 'path'; +import os from 'os'; +import { promises as fs } from 'fs'; +import { fileURLToPath } from 'url'; import { pool } from '../db/pool.js'; import { authMiddleware, AuthenticatedRequest } from '../middleware/auth.js'; import { getStorageProvider } from '../services/storage/factory.js'; +import { spawn } from 'child_process'; const router = Router(); +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); +const AUDIO_DIR = path.join(__dirname, '../../public/audio'); const upload = multer({ storage: multer.memoryStorage(), @@ -31,6 +38,93 @@ const upload = multer({ } }); +const findWhisperExecutable = async (): Promise => { + if (process.env.WHISPER_CMD) return process.env.WHISPER_CMD; + const customPath = process.env.WHISPER_PATH; + if (customPath) { + const candidate = path.join(customPath, 'whisper'); + try { + await fs.access(candidate); + return candidate; + } catch { + // ignore + } + } + + const pathEntries = (process.env.PATH || '').split(path.delimiter); + for (const entry of pathEntries) { + const candidate = path.join(entry, 'whisper'); + try { + await fs.access(candidate); + return candidate; + } catch { + // ignore + } + } + return null; +}; + +const transcribeWithWhisper = async (buffer: Buffer, originalFilename: string, signal?: AbortSignal): Promise => { + const whisperCmd = await findWhisperExecutable(); + if (!whisperCmd) return null; + + const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'whisper-')); + const ext = path.extname(originalFilename) || '.mp3'; + const inputPath = path.join(tempDir, `input${ext}`); + const outputDir = path.join(tempDir, 'out'); + + try { + await fs.mkdir(outputDir, { recursive: true }); + await fs.writeFile(inputPath, buffer); + + const args = [ + inputPath, + '--model', 'base', + '--output_format', 'txt', + '--output_dir', outputDir, + '--fp16', 'False' + ]; + + await new Promise((resolve, reject) => { + const proc = spawn(whisperCmd, args, { stdio: 'ignore' }); + const handleAbort = () => { + proc.kill('SIGTERM'); + reject(new Error('Transcription cancelled')); + }; + if (signal) { + if (signal.aborted) { + handleAbort(); + return; + } + signal.addEventListener('abort', handleAbort, { once: true }); + } + proc.on('error', reject); + proc.on('close', (code) => { + if (signal) { + signal.removeEventListener('abort', handleAbort); + } + if (code === 0) resolve(); + else reject(new Error(`Whisper exited with code ${code}`)); + }); + }); + + const files = await fs.readdir(outputDir); + const txtFile = files.find((file) => file.endsWith('.txt')); + if (!txtFile) return null; + const text = await fs.readFile(path.join(outputDir, txtFile), 'utf8'); + return text.trim() || null; + } catch (error) { + console.warn('Whisper transcription failed:', error); + return null; + } finally { + try { + await fs.rm(tempDir, { recursive: true, force: true }); + } catch { + // ignore + } + } +}; + // Get user's reference tracks router.get('/', authMiddleware, async (req: AuthenticatedRequest, res: Response) => { try { @@ -72,6 +166,7 @@ router.post('/', authMiddleware, upload.single('audio'), async (req: Authenticat const storage = getStorageProvider(); await storage.upload(key, req.file.buffer, req.file.mimetype); const audioUrl = storage.getPublicUrl(key); + const whisperAvailable = Boolean(await findWhisperExecutable()); // Parse tags from request body if provided const tags = req.body.tags ? JSON.parse(req.body.tags) : null; @@ -87,7 +182,8 @@ router.post('/', authMiddleware, upload.single('audio'), async (req: Authenticat track: { ...result.rows[0], audio_url: audioUrl - } + }, + whisper_available: whisperAvailable }); } catch (error) { console.error('Upload reference track error:', error); @@ -153,6 +249,44 @@ router.patch('/:id', authMiddleware, async (req: AuthenticatedRequest, res: Resp } }); +// Transcribe a reference track with whisper (if available) +router.post('/:id/transcribe', authMiddleware, async (req: AuthenticatedRequest, res: Response) => { + try { + const whisperCmd = await findWhisperExecutable(); + if (!whisperCmd) { + res.status(404).json({ error: 'Whisper not available' }); + return; + } + + const result = await pool.query( + 'SELECT user_id, filename, storage_key FROM reference_tracks WHERE id = $1', + [req.params.id] + ); + if (result.rows.length === 0) { + res.status(404).json({ error: 'Track not found' }); + return; + } + if (result.rows[0].user_id !== req.user!.id) { + res.status(403).json({ error: 'Access denied' }); + return; + } + + const audioPath = path.join(AUDIO_DIR, result.rows[0].storage_key); + const buffer = await fs.readFile(audioPath); + const controller = new AbortController(); + + req.on('close', () => controller.abort()); + + const lyrics = await transcribeWithWhisper(buffer, result.rows[0].filename, controller.signal); + if (controller.signal.aborted) return; + + res.json({ lyrics: lyrics || '' }); + } catch (error) { + console.error('Transcribe reference track error:', error); + res.status(500).json({ error: 'Failed to transcribe' }); + } +}); + // Delete a reference track router.delete('/:id', authMiddleware, async (req: AuthenticatedRequest, res: Response) => { try {