import { useState, useEffect, useCallback, useRef } from 'react'; import { Download, Trash2, X, ChevronDown, ChevronUp, Settings2, Eye } from 'lucide-react'; import { Button } from '../../ui/button'; import { useModelAndProvider } from '../../ModelAndProviderContext'; import { defineMessages, useIntl } from '../../../i18n'; import { listLocalModels, syncFeaturedModels, downloadHfModel, getLocalModelDownloadProgress, cancelLocalModelDownload, deleteLocalModel, setConfigProvider, type DownloadProgress, type LocalModelResponse, } from '../../../api'; import { HuggingFaceModelSearch } from './HuggingFaceModelSearch'; import { ModelSettingsPanel } from './ModelSettingsPanel'; import { Dialog, DialogContent, DialogHeader, DialogTitle } from '../../ui/dialog'; const i18n = defineMessages({ title: { id: 'localInferenceSettings.title', defaultMessage: 'Local Inference Models', }, description: { id: 'localInferenceSettings.description', defaultMessage: 'Download and manage local LLM models for inference without API keys. Search HuggingFace for any GGUF model or use the featured picks below.', }, downloading: { id: 'localInferenceSettings.downloading', defaultMessage: 'Downloading', }, downloadedModels: { id: 'localInferenceSettings.downloadedModels', defaultMessage: 'Downloaded Models', }, featuredModels: { id: 'localInferenceSettings.featuredModels', defaultMessage: 'Featured Models', }, recommended: { id: 'localInferenceSettings.recommended', defaultMessage: 'Recommended', }, download: { id: 'localInferenceSettings.download', defaultMessage: 'Download', }, showRecommendedOnly: { id: 'localInferenceSettings.showRecommendedOnly', defaultMessage: 'Show recommended only', }, showAllFeatured: { id: 'localInferenceSettings.showAllFeatured', defaultMessage: 'Show all featured ({count} more)', }, modelSettings: { id: 'localInferenceSettings.modelSettings', defaultMessage: 'Model Settings', }, noModels: { id: 'localInferenceSettings.noModels', defaultMessage: 'No models available', }, downloadProgress: { id: 'localInferenceSettings.downloadProgress', defaultMessage: '{downloaded} / {total} ({percent}%)', }, remaining: { id: 'localInferenceSettings.remaining', defaultMessage: '{time} remaining', }, downloadFailed: { id: 'localInferenceSettings.downloadFailed', defaultMessage: 'Download failed', }, deleteConfirm: { id: 'localInferenceSettings.deleteConfirm', defaultMessage: 'Delete this model? You can re-download it later.', }, modelSettingsTitle: { id: 'localInferenceSettings.modelSettingsTitle', defaultMessage: 'Model settings', }, vision: { id: 'localInferenceSettings.vision', defaultMessage: 'Vision', }, visionEncoderDownloading: { id: 'localInferenceSettings.visionEncoderDownloading', defaultMessage: 'Vision encoder downloading…', }, visionEncoderNotDownloaded: { id: 'localInferenceSettings.visionEncoderNotDownloaded', defaultMessage: 'Vision encoder not downloaded', }, }); const VisionBadge = ({ model, intl, }: { model: LocalModelResponse; intl: ReturnType; }) => { if (!model.vision_capable) return null; const mmproj = model.mmproj_status; const isDownloaded = mmproj?.state === 'Downloaded'; const isDownloading = mmproj?.state === 'Downloading'; if (isDownloaded) { return ( {intl.formatMessage(i18n.vision)} ); } if (isDownloading) { const percent = mmproj && 'progress_percent' in mmproj ? Math.round(mmproj.progress_percent) : null; return ( {intl.formatMessage(i18n.visionEncoderDownloading)} {percent != null && ` ${percent}%`} ); } return ( {intl.formatMessage(i18n.vision)} ); }; const formatBytes = (bytes: number): string => { if (bytes < 1024) return `${bytes}B`; if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(0)}KB`; if (bytes < 1024 * 1024 * 1024) return `${(bytes / (1024 * 1024)).toFixed(0)}MB`; return `${(bytes / (1024 * 1024 * 1024)).toFixed(1)}GB`; }; export const LocalInferenceSettings = () => { const intl = useIntl(); const [models, setModels] = useState([]); const [downloads, setDownloads] = useState>(new Map()); const [showAllFeatured, setShowAllFeatured] = useState(false); const [settingsOpenFor, setSettingsOpenFor] = useState(null); const { currentModel, currentProvider, refreshCurrentModelAndProvider } = useModelAndProvider(); const downloadSectionRef = useRef(null); const activePolls = useRef(new Set()); const selectedModelId = currentProvider === 'local' ? currentModel : null; const loadModels = useCallback(async (): Promise => { try { await syncFeaturedModels(); const response = await listLocalModels(); if (response.data) { setModels(response.data); response.data.forEach((model) => { if (model.status.state === 'Downloading') { pollDownloadProgress(model.id); } }); return response.data; } } catch (error) { console.error('Failed to load models:', error); } return undefined; // eslint-disable-next-line react-hooks/exhaustive-deps }, []); useEffect(() => { loadModels(); // eslint-disable-next-line react-hooks/exhaustive-deps }, []); // Poll model list while any vision encoder is downloading useEffect(() => { const hasDownloadingMmproj = models.some( (m) => m.vision_capable && m.mmproj_status?.state === 'Downloading' ); if (!hasDownloadingMmproj) return; const interval = setInterval(() => { loadModels(); }, 2000); return () => clearInterval(interval); }, [models, loadModels]); const selectModel = async (modelId: string) => { try { await setConfigProvider({ body: { provider: 'local', model: modelId }, throwOnError: true, }); await refreshCurrentModelAndProvider(); } catch (error) { console.error('Failed to select model:', error); } }; const startFeaturedDownload = async (modelId: string) => { const model = models.find((m) => m.id === modelId); if (!model) return; try { await downloadHfModel({ body: { spec: model.id } }); pollDownloadProgress(modelId); scrollToDownloads(); } catch (error) { console.error('Failed to start download:', error); } }; const scrollToDownloads = useCallback(() => { requestAnimationFrame(() => { downloadSectionRef.current?.scrollIntoView({ behavior: 'smooth', block: 'nearest' }); }); }, []); const pollDownloadProgress = (modelId: string) => { if (activePolls.current.has(modelId)) return; activePolls.current.add(modelId); const stopPolling = (interval: ReturnType) => { clearInterval(interval); activePolls.current.delete(modelId); }; const interval = setInterval(async () => { try { const response = await getLocalModelDownloadProgress({ path: { model_id: modelId } }); if (response.data) { const progress = response.data; setDownloads((prev) => new Map(prev).set(modelId, progress)); if (progress.status === 'completed') { stopPolling(interval); setDownloads((prev) => { const next = new Map(prev); next.delete(modelId); return next; }); await loadModels(); await selectModel(modelId); } else if (progress.status === 'failed' || progress.status === 'cancelled') { stopPolling(interval); setDownloads((prev) => { const next = new Map(prev); next.delete(modelId); return next; }); await loadModels(); } } else { stopPolling(interval); } } catch { stopPolling(interval); } }, 1000); }; const cancelDownload = async (modelId: string) => { try { await cancelLocalModelDownload({ path: { model_id: modelId } }); setDownloads((prev) => { const next = new Map(prev); next.delete(modelId); return next; }); await loadModels(); } catch (error) { console.error('Failed to cancel download:', error); } }; const handleDeleteModel = async (modelId: string) => { if (!window.confirm(intl.formatMessage(i18n.deleteConfirm))) return; try { await deleteLocalModel({ path: { model_id: modelId } }); const updatedModels = await loadModels(); if (selectedModelId === modelId && updatedModels) { const remainingDownloaded = updatedModels.filter( (m) => m.id !== modelId && m.status.state === 'Downloaded' ); if (remainingDownloaded.length > 0) { selectModel(remainingDownloaded[0].id); } } } catch (error) { console.error('Failed to delete model:', error); } }; const handleHfDownloadStarted = (modelId: string) => { pollDownloadProgress(modelId); loadModels(); scrollToDownloads(); }; const isDownloaded = (model: LocalModelResponse) => model.status.state === 'Downloaded'; const isNotDownloaded = (model: LocalModelResponse) => model.status.state === 'NotDownloaded' && !downloads.has(model.id); const downloadedModels = models.filter(isDownloaded); const notDownloadedModels = models.filter(isNotDownloaded); const recommendedModels = notDownloadedModels.filter((m) => m.recommended); const displayedFeatured = showAllFeatured ? notDownloadedModels : recommendedModels; const showFeaturedToggle = notDownloadedModels.length > recommendedModels.length; return (

{intl.formatMessage(i18n.title)}

{intl.formatMessage(i18n.description)}

{/* Active Downloads */} {downloads.size > 0 && (

{intl.formatMessage(i18n.downloading)}

{Array.from(downloads.entries()).map(([modelId, progress]) => { if (progress.status === 'completed') return null; return (

{modelId} {progress.status === 'downloading' && ( )}

{progress.status === 'downloading' && (

{intl.formatMessage(i18n.downloadProgress, { downloaded: formatBytes(progress.bytes_downloaded), total: formatBytes(progress.total_bytes), percent: progress.progress_percent.toFixed(0), })} {progress.eta_seconds != null && progress.eta_seconds > 0 && ( {intl.formatMessage(i18n.remaining, { time: progress.eta_seconds < 60 ? `${Math.round(progress.eta_seconds)}s` : `${Math.round(progress.eta_seconds / 60)}m`, })} )} {progress.speed_bps != null && progress.speed_bps > 0 && ( {formatBytes(progress.speed_bps)}/s )}

)} {progress.status === 'failed' && (

{progress.error || intl.formatMessage(i18n.downloadFailed)}

)}

); })}

)} {/* Downloaded Models */} {downloadedModels.length > 0 && (

{intl.formatMessage(i18n.downloadedModels)}

{downloadedModels.map((model) => { const isSelected = selectedModelId === model.id; return (

selectModel(model.id)} className="cursor-pointer" /> {model.id} {formatBytes(model.size_bytes)} {model.recommended && ( {intl.formatMessage(i18n.recommended)} )}

); })}

)} {/* Featured Models (not yet downloaded) */} {displayedFeatured.length > 0 && (

{intl.formatMessage(i18n.featuredModels)}

{displayedFeatured.map((model) => (

{model.id}

{formatBytes(model.size_bytes)} {model.recommended && ( {intl.formatMessage(i18n.recommended)} )}

))}

{showFeaturedToggle && ( )}

)} {/* HuggingFace Search */}

m.status.state === 'Downloaded').map((m) => m.id)) } />

{models.length === 0 && (

{intl.formatMessage(i18n.noModels)}

)}

); };