/** * NPM package installer for virtual file systems. * * This module fetches packages from the npm registry and populates * a virtual node_modules directory structure. */ import * as semver from "semver"; import type { Files } from "./types"; const NPM_REGISTRY = "https://registry.npmjs.org"; const DEFAULT_TIMEOUT_MS = 30000; // 30 seconds /** * Fetch with a timeout. * Throws an error if the request takes longer than the specified timeout. */ async function fetchWithTimeout( url: string, options: RequestInit = {}, timeoutMs = DEFAULT_TIMEOUT_MS ): Promise { const controller = new AbortController(); const timeoutId = setTimeout(() => controller.abort(), timeoutMs); try { return await fetch(url, { ...options, signal: controller.signal }); } catch (error) { if (error instanceof Error && error.name === "AbortError") { throw new Error(`Request timed out after ${timeoutMs}ms: ${url}`); } throw error; } finally { clearTimeout(timeoutId); } } interface PackageJson { name: string; version: string; main?: string; module?: string; exports?: unknown; dependencies?: Record; peerDependencies?: Record; devDependencies?: Record; dist?: { tarball: string; integrity?: string; }; } interface NpmPackageMetadata { name: string; "dist-tags": Record; versions: Record; } interface InstallOptions { /** * Include devDependencies (default: false) */ dev?: boolean; /** * Registry URL (default: https://registry.npmjs.org) */ registry?: string; } interface InstallResult { /** * Files with node_modules populated */ files: Files; /** * Packages that were installed */ installed: string[]; /** * Warnings encountered during installation */ warnings: string[]; } /** * Install npm dependencies into a virtual file system. * * Reads the package.json from the files, resolves all dependencies, * and populates node_modules with the package contents. * * @param files - Virtual file system containing package.json * @param options - Installation options * @returns Files with node_modules populated */ export async function installDependencies( files: Files, options: InstallOptions = {} ): Promise { const { dev = false, registry = NPM_REGISTRY } = options; const result: InstallResult = { files: { ...files }, installed: [], warnings: [] }; // Read package.json const packageJsonContent = files["package.json"]; if (!packageJsonContent) { return result; // No package.json, nothing to install } let packageJson: PackageJson; try { packageJson = JSON.parse(packageJsonContent) as PackageJson; } catch { result.warnings.push("Failed to parse package.json"); return result; } // Collect dependencies to install const depsToInstall: Record = { ...packageJson.dependencies, ...(dev ? packageJson.devDependencies : {}) }; if (Object.keys(depsToInstall).length === 0) { return result; // No dependencies to install } // Track installed packages to avoid duplicates const installedPackages = new Map(); // name -> version // Track in-progress installations to avoid duplicate work const inProgress = new Map>(); // Install all dependencies in parallel await Promise.all( Object.entries(depsToInstall).map(([name, versionRange]) => installPackage( name, versionRange, result, installedPackages, inProgress, registry ) ) ); return result; } /** * Install a single package and its dependencies recursively. */ async function installPackage( name: string, versionRange: string, result: InstallResult, installedPackages: Map, inProgress: Map>, registry: string ): Promise { // Skip if already installed if (installedPackages.has(name)) { return; } // If installation is already in progress, wait for it const existing = inProgress.get(name); if (existing) { return existing; } // Create the installation promise const installPromise = (async () => { try { // Fetch package metadata from registry const metadata = await fetchPackageMetadata(name, registry); // Resolve version from range const version = resolveVersion(versionRange, metadata); if (!version) { result.warnings.push( `Could not resolve version for ${name}@${versionRange}` ); return; } // Get the specific version metadata const versionMetadata = metadata.versions[version]; if (!versionMetadata) { result.warnings.push(`Version ${version} not found for ${name}`); return; } // Mark as installed (before fetching to prevent cycles) installedPackages.set(name, version); result.installed.push(`${name}@${version}`); // Fetch and extract the package tarball const packageFiles = await fetchPackageFiles(name, versionMetadata); // Add files to node_modules for (const [filePath, content] of Object.entries(packageFiles)) { result.files[`node_modules/${name}/${filePath}`] = content; } // Install dependencies in parallel const deps = versionMetadata.dependencies ?? {}; await Promise.all( Object.entries(deps).map(([depName, depVersion]) => installPackage( depName, depVersion, result, installedPackages, inProgress, registry ) ) ); } catch (error) { const message = error instanceof Error ? error.message : String(error); result.warnings.push(`Failed to install ${name}: ${message}`); } })(); // Track in progress inProgress.set(name, installPromise); try { await installPromise; } finally { inProgress.delete(name); } } /** * Fetch package metadata from npm registry. */ async function fetchPackageMetadata( name: string, registry: string ): Promise { // Handle scoped packages const encodedName = name.startsWith("@") ? `@${encodeURIComponent(name.slice(1))}` : name; const url = `${registry}/${encodedName}`; const response = await fetchWithTimeout(url, { headers: { // Use abbreviated metadata to avoid fetching megabytes of version data Accept: "application/vnd.npm.install-v1+json; q=1.0, application/json; q=0.8" } }); if (!response.ok) { throw new Error(`Failed to fetch package metadata: ${response.status}`); } return (await response.json()) as NpmPackageMetadata; } /** * Resolve a semver range to a specific version. */ function resolveVersion( range: string, metadata: NpmPackageMetadata ): string | undefined { // Handle special cases if (range === "latest" || range === "*") { return metadata["dist-tags"]["latest"]; } // Handle exact versions if (metadata.versions[range]) { return range; } // Handle dist-tags (e.g., "next", "beta") if (metadata["dist-tags"][range]) { return metadata["dist-tags"][range]; } // Use semver.maxSatisfying to find the best matching version const versions = Object.keys(metadata.versions); const match = semver.maxSatisfying(versions, range); return match ?? undefined; } /** * Fetch and extract package files from npm tarball. */ async function fetchPackageFiles( name: string, metadata: PackageJson ): Promise> { const tarballUrl = metadata.dist?.tarball; if (!tarballUrl) { throw new Error(`No tarball URL for ${name}`); } // Fetch the tarball (use longer timeout for potentially large packages) const response = await fetchWithTimeout( tarballUrl, {}, DEFAULT_TIMEOUT_MS * 2 ); if (!response.ok) { throw new Error(`Failed to fetch tarball: ${response.status}`); } // Get the tarball as array buffer const buffer = await response.arrayBuffer(); // Extract the tarball (npm tarballs are gzipped tar files) return extractTarball(new Uint8Array(buffer)); } /** * Extract files from a gzipped tarball. * * npm packages are distributed as .tgz files (gzipped tar). * The contents are in a "package/" directory. */ async function extractTarball( data: Uint8Array ): Promise> { // Decompress gzip const decompressed = await decompress(data); // Parse tar return parseTar(decompressed); } /** * Decompress gzip data using DecompressionStream. */ async function decompress(data: Uint8Array): Promise { // Use DecompressionStream (available in Workers and modern browsers) const ds = new DecompressionStream("gzip"); const writer = ds.writable.getWriter(); const reader = ds.readable.getReader(); // Write compressed data writer.write(data as Uint8Array).catch(() => {}); writer.close().catch(() => {}); // Read decompressed data const chunks: Uint8Array[] = []; let totalLength = 0; while (true) { const { done, value } = await reader.read(); if (done) break; chunks.push(value); totalLength += value.length; } // Concatenate chunks const result = new Uint8Array(totalLength); let offset = 0; for (const chunk of chunks) { result.set(chunk, offset); offset += chunk.length; } return result; } /** * Parse a tar archive and extract text files. * * TAR format: * - 512-byte header blocks * - File content (padded to 512 bytes) * - Two empty blocks at the end */ function parseTar(data: Uint8Array): Record { const files: Record = {}; const textDecoder = new TextDecoder(); let offset = 0; while (offset < data.length - 512) { // Read header const header = data.slice(offset, offset + 512); // Check for empty block (end of archive) if (header.every((b) => b === 0)) { break; } // Parse header fields const name = readString(header, 0, 100); const sizeStr = readString(header, 124, 12); const typeFlag = header[156]; // Parse size (octal) const size = parseInt(sizeStr.trim(), 8) || 0; // Move past header offset += 512; // Only process regular files (type '0' or '\0') if ((typeFlag === 48 || typeFlag === 0) && size > 0) { // Read file content const content = data.slice(offset, offset + size); // Remove "package/" prefix from npm tarballs let filePath = name; if (filePath.startsWith("package/")) { filePath = filePath.slice(8); } // Only include text files (skip binary files) if (isTextFile(filePath)) { try { files[filePath] = textDecoder.decode(content); } catch { // Skip files that can't be decoded as text } } } // Move to next block (content is padded to 512 bytes) offset += Math.ceil(size / 512) * 512; } return files; } /** * Read a null-terminated string from a buffer. */ function readString( buffer: Uint8Array, offset: number, length: number ): string { const bytes = buffer.slice(offset, offset + length); const nullIndex = bytes.indexOf(0); const relevantBytes = nullIndex >= 0 ? bytes.slice(0, nullIndex) : bytes; return new TextDecoder().decode(relevantBytes); } /** * Check if a file path is likely a text file. */ function isTextFile(path: string): boolean { const textExtensions = [ ".js", ".mjs", ".cjs", ".ts", ".mts", ".cts", ".tsx", ".jsx", ".json", ".md", ".txt", ".css", ".html", ".yml", ".yaml", ".toml", ".xml", ".svg", ".map", ".d.ts", ".d.mts", ".d.cts" ]; // Check common config files without extensions const configFiles = [ "LICENSE", "README", "CHANGELOG", "package.json", "tsconfig.json", ".npmignore", ".gitignore" ]; const fileName = path.split("/").pop() ?? ""; if ( configFiles.some((f) => fileName.toUpperCase().startsWith(f.toUpperCase())) ) { return true; } return textExtensions.some((ext) => path.toLowerCase().endsWith(ext)); } /** * Check if files contain a package.json with dependencies that need installing. */ export function hasDependencies(files: Files): boolean { const packageJson = files["package.json"]; if (!packageJson) return false; try { const pkg = JSON.parse(packageJson); const deps = pkg.dependencies ?? {}; return Object.keys(deps).length > 0; } catch { return false; } }