From 9fd654298aaa5e08d8bf78927125164f3e846860 Mon Sep 17 00:00:00 2001 From: aculix <35836838+aculix@users.noreply.github.com> Date: Tue, 29 Sep 2026 16:25:50 +0530 Subject: [PATCH] Fix first-run setup timing out on the kernel download On a Mac without the Linux kernel, setup ran `container system start --enable-kernel-install` with a 120 s limit. That command downloads a 697 MB archive without printing anything, so most connections hit the limit and the window showed `timedOut([...])`. - Start the service with --disable-kernel-install (10 min: a first start still pulls the 69 MB base filesystem). - Download the kernel in its own step with progress, then hand the file to `container system kernel set --tar ... --digest ...`, which checks it against the runtime's digest. - Opening the setup window no longer starts the download; Set Up does. - Timeouts and runner errors read as sentences. - A rebuild deletes the image it replaced, which frees about 2 GB each time, and deletes or stops the builder VM unless it was already running (it kept about 2.8 GB of memory). - Setup asks for 10 GB free, measured on a real setup. --- README.md | 7 +- Sources/Knotchy/AppDelegate.swift | 1 + Sources/Knotchy/MenuBar.swift | 3 +- Sources/Knotchy/NotchViews.swift | 15 +- .../Backend/ContainerCLIBackend.swift | 179 ++++++++++++++++-- .../Session/SessionController.swift | 9 +- Sources/KnotchyCore/Setup/FileDownload.swift | 100 ++++++++++ .../KnotchyCore/Setup/RuntimeInstaller.swift | 93 +-------- Sources/KnotchyCore/Setup/SetupFlow.swift | 37 +++- .../ContainerCLIBackendSetupTests.swift | 163 ++++++++++++++++ .../SessionSetupRunTests.swift | 32 ++++ .../KnotchyCoreTests/SessionSetupTests.swift | 9 + Tests/KnotchyCoreTests/SetupFlowTests.swift | 39 +++- 13 files changed, 559 insertions(+), 128 deletions(-) create mode 100644 Sources/KnotchyCore/Setup/FileDownload.swift create mode 100644 Tests/KnotchyCoreTests/ContainerCLIBackendSetupTests.swift diff --git a/README.md b/README.md index 4c34fe0..a1f0b74 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ The VM can reach the internet. It can't reach your Mac or anything else on your - A Mac with Apple silicon - macOS 26 or later -- About 3 GB of free space for the first setup +- About 10 GB of free space for the first setup. Around 5 GB stays in use once it's done. - An administrator password, once, for Apple's installer ## Install @@ -31,7 +31,7 @@ The VM can reach the internet. It can't reach your Mac or anything else on your - It downloads Apple's [`container`](https://github.com/apple/container) runtime (version 1.4.1, 118 MB), checks it against the SHA-256 built into Knotchy, and opens it in Apple's Installer. Click through Installer and enter your password there. Knotchy never asks for administrator rights itself. - - It starts the runtime, which downloads a Linux kernel the first time. + - It starts the runtime and downloads its Linux kernel, about 700 MB. - It builds the browser VM image on your Mac. That takes about 3 minutes and downloads about 225 MB. 5. When every row has a green tick, click **Done**. @@ -88,7 +88,8 @@ Proxies on your Mac or your local network are refused, because a page could reac Passwords and VPN keys are kept in your keychain. VM shows the runtime version and when the image was built, and has **Rebuild Now**. Rebuilding picks up WebKit -security fixes, and Knotchy suggests it once the image is two weeks old. +security fixes, and Knotchy suggests it once the image is two weeks old. After a build, Knotchy deletes the image it +replaced and the builder VM, unless that builder was already running before the build. ## How it works diff --git a/Sources/Knotchy/AppDelegate.swift b/Sources/Knotchy/AppDelegate.swift index 5ad9081..1d20b75 100644 --- a/Sources/Knotchy/AppDelegate.swift +++ b/Sources/Knotchy/AppDelegate.swift @@ -158,6 +158,7 @@ final class AppDelegate: NSObject, NSApplicationDelegate { quitting = true setupFlow.cancel() if setupFlow.isDownloading { SystemRuntimeInstaller.removeLeftovers() } + if controller.state == .settingUp(.kernelMissing) { ContainerCLIBackend.removeKernelDownloads() } // Not Now mid-build closes the window before the build ends. if setupFlow.allRequiredDone { UserDefaults.standard.set(true, forKey: SetupLaunchRule.completedKey) } // Synchronous on purpose: terminate: often runs inside a non-reentrant main-queue block; awaiting here deadlocked. diff --git a/Sources/Knotchy/MenuBar.swift b/Sources/Knotchy/MenuBar.swift index b454561..f6505e4 100644 --- a/Sources/Knotchy/MenuBar.swift +++ b/Sources/Knotchy/MenuBar.swift @@ -205,7 +205,8 @@ struct SessionSummary { switch issue { case .cliMissing: "Installing the container runtime…" case .cliOutdated: "Updating the container runtime…" - case .serviceStopped, .kernelMissing: "Starting the container runtime…" + case .serviceStopped: "Starting the container runtime…" + case .kernelMissing: "Downloading the Linux kernel…" default: "Building the VM…" } } diff --git a/Sources/Knotchy/NotchViews.swift b/Sources/Knotchy/NotchViews.swift index 5c98876..083a0a7 100644 --- a/Sources/Knotchy/NotchViews.swift +++ b/Sources/Knotchy/NotchViews.swift @@ -553,7 +553,8 @@ private enum Setup { static func symbol(_ issue: SetupIssue) -> String { switch issue { case .cliMissing, .cliOutdated: "shippingbox" - case .serviceStopped, .kernelMissing: "power" + case .serviceStopped: "power" + case .kernelMissing: "arrow.down.circle" case .imageMissing, .imageOutdated: "square.and.arrow.down" case .imageStale: "arrow.triangle.2.circlepath" case .lowDisk: "internaldrive" @@ -564,7 +565,8 @@ private enum Setup { switch issue { case .cliMissing: "Knotchy needs Apple's container runtime" case .cliOutdated: "Knotchy needs a newer container runtime" - case .serviceStopped, .kernelMissing: "The container service isn't running" + case .serviceStopped: "The container service isn't running" + case .kernelMissing: "The container runtime needs its Linux kernel" case .imageMissing, .imageOutdated: "The browser VM image needs to be built" case .imageStale: "Updating the browser VM" case .lowDisk: "Not enough disk space" @@ -575,7 +577,8 @@ private enum Setup { switch issue { case .cliMissing: "Installing the container runtime…" case .cliOutdated: "Updating the container runtime…" - case .serviceStopped, .kernelMissing: "Starting the container runtime…" + case .serviceStopped: "Starting the container runtime…" + case .kernelMissing: "Setting up the container runtime…" default: title(issue) } } @@ -584,7 +587,8 @@ private enum Setup { switch issue { case .cliMissing: "Set it up in one step; the link opens when it's done." case .cliOutdated: "Update it in one step; the link opens when it's done." - case .serviceStopped, .kernelMissing: "Start it to run the isolated browser." + case .serviceStopped: "Start it to run the isolated browser." + case .kernelMissing: "It's a 700 MB download. The link opens when it's done." case .imageMissing, .imageOutdated: "This takes about 3 minutes and downloads about 225 MB." case .imageStale: "Rebuilding gets the latest WebKit security fixes." case .lowDisk(let free): "Only \(free / 1_000_000_000) GB free; a session needs about 2 GB." @@ -594,7 +598,8 @@ private enum Setup { static func action(_ issue: SetupIssue) -> String? { switch issue { case .cliMissing, .cliOutdated: "Set Up…" - case .serviceStopped, .kernelMissing: "Start Service" + case .serviceStopped: "Start Service" + case .kernelMissing: "Download" case .imageMissing, .imageOutdated: "Build Image" case .imageStale: "Update Now" case .lowDisk: nil diff --git a/Sources/KnotchyCore/Backend/ContainerCLIBackend.swift b/Sources/KnotchyCore/Backend/ContainerCLIBackend.swift index 464bf70..74c6893 100644 --- a/Sources/KnotchyCore/Backend/ContainerCLIBackend.swift +++ b/Sources/KnotchyCore/Backend/ContainerCLIBackend.swift @@ -3,6 +3,8 @@ import Synchronization public final class ContainerCLIBackend: VMBackend { public static let image = "local/knotchy-vm:latest" + /// The image a rebuild replaces, named so it can be deleted afterwards. + static let previousImage = "local/knotchy-vm:previous" public static let staleAfterDays = 14 private let cli: CLIRunner @@ -271,30 +273,164 @@ public final class ContainerCLIBackend: VMBackend { // MARK: Setup actions + /// A first start also pulls the runtime's base filesystem, about 70 MB. + static let serviceTimeout: TimeInterval = 10 * 60 + /// A first build downloads about 470 MB: the builder, the Debian base and its packages. + static let buildTimeout: TimeInterval = 60 * 60 + static let kernelFolderPrefix = "knotchy-kernel-" + public func fix(_ issue: SetupIssue, output: @escaping @Sendable (String) -> Void) async throws { switch issue { - case .serviceStopped, .kernelMissing: - let result = try await cli.run(["system", "start", "--enable-kernel-install"], timeout: 120, line: output) + case .serviceStopped: + // The kernel is its own step: `--enable-kernel-install` downloads 700 MB without showing progress. + let result = try await step(["system", "start", "--disable-kernel-install"], timeout: Self.serviceTimeout, + output: output, named: "Starting the container service") guard result.status == 0 else { throw VMBackendError.failed("Couldn't start the service: \(result.lastLine)") } + case .kernelMissing: + try await installKernel(output: output) case .imageMissing, .imageOutdated, .imageStale: - guard let vmSources else { throw VMBackendError.failed("The VM sources are missing from the app.") } - let now = Date() - let day = DateFormatter() - day.dateFormat = "yyyy-MM-dd" - let args = [ - "build", "--cpus", "4", "--memory", "4g", "--dns", "1.1.1.1", "--pull", - "--build-arg", "APT_REFRESH=\(day.string(from: now))", - "--build-arg", "BUILD_DATE=\(ISO8601DateFormatter().string(from: now))", - "-f", vmSources.appendingPathComponent("Containerfile").path, - "-t", Self.image, vmSources.path, - ] - let result = try await cli.run(args, timeout: 15 * 60, line: output) - guard result.status == 0 else { throw VMBackendError.failed("The image build failed: \(result.lastLine)") } + try await buildImage(output: output) case .cliMissing, .cliOutdated, .lowDisk: throw VMBackendError.failed("There's no automatic fix for this.") } } + /// Leaves nothing behind but the new image: not the one it replaced, and not a builder it started. + private func buildImage(output: @escaping @Sendable (String) -> Void) async throws { + guard let vmSources else { throw VMBackendError.failed("The VM sources are missing from the app.") } + let now = Date() + let day = DateFormatter() + day.dateFormat = "yyyy-MM-dd" + let args = [ + "build", "--cpus", "4", "--memory", "4g", "--dns", "1.1.1.1", "--pull", + "--build-arg", "APT_REFRESH=\(day.string(from: now))", + "--build-arg", "BUILD_DATE=\(ISO8601DateFormatter().string(from: now))", + "-f", vmSources.appendingPathComponent("Containerfile").path, + "-t", Self.image, vmSources.path, + ] + let builderBefore = (try? await cli.run(["builder", "status", "--format", "json"], timeout: 15)) + .flatMap { Self.builderState(in: $0.text) } + // The runtime keeps a replaced image's files, about 2 GB, until an image is deleted. Deleting the + // old one by name frees them, along with anything earlier builds left. + let named = (try? await cli.run(["image", "tag", Self.image, Self.previousImage], timeout: 30))?.status == 0 + let built: Result + do { + built = .success(try await step(args, timeout: Self.buildTimeout, output: output, named: "Building the VM image")) + } catch { + built = .failure(error) + } + // After a failed build the old image keeps its usual name too, so this deletes nothing it needs. + if named { _ = try? await cli.run(["image", "delete", Self.previousImage], timeout: 5 * 60) } + // The builder VM keeps about 3 GB of memory and a 4 GB cache. One the user had running stays as it was. + switch builderBefore { + case .absent?: _ = try? await cli.run(["builder", "delete", "--force"], timeout: 60) + case .stopped?: _ = try? await cli.run(["builder", "stop"], timeout: 60) + case .running?, nil: break + } + let result = try built.get() + guard result.status == 0 else { throw VMBackendError.failed("The image build failed: \(result.lastLine)") } + } + + enum BuilderState: Equatable { + case absent + case stopped + case running + } + + /// From `builder status --format json`: `[]` when there's no builder. + static func builderState(in json: String) -> BuilderState? { + struct Entry: Decodable { + struct Status: Decodable { + let state: String? + } + let status: Status? + } + guard let entries = try? JSONDecoder().decode([Entry].self, from: Data(json.utf8)) else { return nil } + guard let first = entries.first else { return .absent } + return first.status?.state == "running" ? .running : .stopped + } + + /// A timeout reads as a sentence, not as the command that ran. + private func step(_ args: [String], timeout: TimeInterval, output: @escaping @Sendable (String) -> Void, + named name: String) async throws -> CLIRunner.Output { + do { + return try await cli.run(args, timeout: timeout, line: output) + } catch CLIRunner.Failure.timedOut { + throw VMBackendError.failed("\(name) took over \(Int(timeout / 60)) minutes, so Knotchy stopped it. Check your connection and try again.") + } + } + + public static func removeKernelDownloads() { + FileDownload.removeFolders(prefix: kernelFolderPrefix) + } + + /// `kernel set --recommended` prints nothing while it downloads 700 MB, so Knotchy downloads the archive + /// itself, with progress, and hands it over. The runtime still checks it against the digest it publishes. + private func installKernel(output: @escaping @Sendable (String) -> Void) async throws { + let properties = try? await cli.run(["system", "property", "list", "--format", "json"], timeout: 15) + guard let kernel = properties.flatMap({ Self.recommendedKernel(in: $0.text) }) else { + let result = try await step(["system", "kernel", "set", "--recommended", "--force"], timeout: 60 * 60, + output: output, named: "Downloading the Linux kernel") + guard result.status == 0 else { throw VMBackendError.failed("Couldn't install the Linux kernel: \(result.lastLine)") } + return + } + Self.removeKernelDownloads() + let folder = URL(fileURLWithPath: NSTemporaryDirectory()) + .appendingPathComponent(Self.kernelFolderPrefix + UUID().uuidString, isDirectory: true) + guard (try? FileManager.default.createDirectory(at: folder, withIntermediateDirectories: false, + attributes: [.posixPermissions: 0o700])) != nil + else { throw VMBackendError.failed("Couldn't make a folder for the Linux kernel download.") } + defer { try? FileManager.default.removeItem(at: folder) } + + let archive = folder.appendingPathComponent(kernel.url.lastPathComponent) + output("Downloading the Linux kernel…") + let throttle = ProgressThrottle() + do { + try await FileDownload.fetch(kernel.url, to: archive) { written, expected in + // One line a percent, so the log keeps what came before the download. + let step = expected.map { Int(written * 100 / max($0, 1)) } ?? Int(written / 10_000_000) + if throttle.advance(to: step) { output(Self.kernelProgress(written, of: expected)) } + } + } catch { + throw VMBackendError.failed("Couldn't download the Linux kernel. Check your connection and try again.") + } + output("Installing the Linux kernel…") + let result = try await step(["system", "kernel", "set", "--tar", archive.path, "--binary", kernel.binaryPath, + "--digest", kernel.digest, "--force"], + timeout: 10 * 60, output: output, named: "Installing the Linux kernel") + guard result.status == 0 else { throw VMBackendError.failed("Couldn't install the Linux kernel: \(result.lastLine)") } + } + + struct RecommendedKernel: Equatable { + let url: URL + let binaryPath: String + let digest: String + } + + /// From `system property list --format json`; nil when the runtime doesn't say. + static func recommendedKernel(in json: String) -> RecommendedKernel? { + struct Properties: Decodable { + struct Kernel: Decodable { + let url: String? + let binaryPath: String? + let digest: String? + } + let kernel: Kernel? + } + guard let kernel = (try? JSONDecoder().decode(Properties.self, from: Data(json.utf8)))?.kernel, + let url = kernel.url.flatMap(URL.init(string:)), + let binaryPath = kernel.binaryPath, !binaryPath.isEmpty, + let digest = kernel.digest, digest.hasPrefix("sha256:") + else { return nil } + return RecommendedKernel(url: url, binaryPath: binaryPath, digest: digest) + } + + static func kernelProgress(_ written: Int64, of expected: Int64?) -> String { + func megabytes(_ bytes: Int64) -> Int { Int((Double(bytes) / 1_000_000).rounded()) } + guard let expected else { return "Downloading the Linux kernel… \(megabytes(written)) MB" } + return "Downloading the Linux kernel… \(megabytes(written)) of \(megabytes(expected)) MB" + } + // MARK: Quit path public func destroyBlocking(ids: [String], deadline: Duration) { @@ -309,3 +445,16 @@ public final class ContainerCLIBackend: VMBackend { } } } + +private final class ProgressThrottle: Sendable { + private let last = Mutex(-1) + + /// True the first time each step is reached. + func advance(to step: Int) -> Bool { + last.withLock { last in + guard step > last else { return false } + last = step + return true + } + } +} diff --git a/Sources/KnotchyCore/Session/SessionController.swift b/Sources/KnotchyCore/Session/SessionController.swift index edbb982..a59522b 100644 --- a/Sources/KnotchyCore/Session/SessionController.swift +++ b/Sources/KnotchyCore/Session/SessionController.swift @@ -557,7 +557,7 @@ public final class SessionController { } } - /// No VM exists yet, so there is nothing to destroy. Starting the service needs no password, so it just happens. + /// No VM exists yet, so there is nothing to destroy. Starting the service or fetching its kernel needs no password, so it just happens. private func needsSetup(_ issue: SetupIssue, for session: Session) { session.retired = true liveIDs.remove(session.id) @@ -752,11 +752,12 @@ public final class SessionController { if mode == .link || pendingURL != nil { transition(to: card) } else { enterIdle() } } + /// A check only starts the service: the kernel is a 700 MB download, so it waits for Set Up. private static func fixes(_ issue: SetupIssue, in mode: RunMode) -> Bool { switch (mode, issue) { case (.window(continuing: true), .serviceStopped), (.window(continuing: true), .kernelMissing), (.window(continuing: true), .imageMissing), (.window(continuing: true), .imageOutdated), - (.check, .serviceStopped), (.check, .kernelMissing): + (.check, .serviceStopped): true default: false @@ -767,7 +768,6 @@ public final class SessionController { private static func sameFix(_ a: SetupIssue, _ b: SetupIssue) -> Bool { func group(_ issue: SetupIssue) -> Int? { switch issue { - case .serviceStopped, .kernelMissing: 0 case .imageMissing, .imageOutdated, .imageStale: 1 default: nil } @@ -925,6 +925,9 @@ public final class SessionController { case StreamError.protocolViolation(let detail): "The VM sent a malformed message: \(detail)." case StreamError.timedOut: "The VM didn't answer in time." case StreamError.refused: "Couldn't connect to the VM." + case CLIRunner.Failure.timedOut: "The container runtime stopped responding. Try again." + case CLIRunner.Failure.notInstalled: "Apple's container runtime isn't installed." + case CLIRunner.Failure.launchFailed(let detail): "Couldn't run the container runtime: \(detail)" default: String(describing: error) } } diff --git a/Sources/KnotchyCore/Setup/FileDownload.swift b/Sources/KnotchyCore/Setup/FileDownload.swift new file mode 100644 index 0000000..9ec0bf8 --- /dev/null +++ b/Sources/KnotchyCore/Setup/FileDownload.swift @@ -0,0 +1,100 @@ +import Foundation + +/// A data task streamed into a file, so a partial download lives only in its own folder. +final class FileDownload: NSObject, URLSessionDataDelegate, @unchecked Sendable { + // Touched only on the session's serial delegate queue. + private var continuation: CheckedContinuation? + /// A completion that arrived before anyone waited: a task cancelled before it started. + private var outcome: Result? + private var handle: FileHandle? + private var written: Int64 = 0 + private var expected: Int64? + private var writeError: (any Error)? + private let destination: URL + private let progress: @Sendable (_ written: Int64, _ expected: Int64?) -> Void + + private init(destination: URL, progress: @escaping @Sendable (Int64, Int64?) -> Void) { + self.destination = destination + self.progress = progress + } + + /// `expected` is nil when the server doesn't say. + static func fetch(_ url: URL, to destination: URL, + progress: @escaping @Sendable (_ written: Int64, _ expected: Int64?) -> Void) async throws { + let delegate = FileDownload(destination: destination, progress: progress) + let queue = OperationQueue() + queue.maxConcurrentOperationCount = 1 + let session = URLSession(configuration: .ephemeral, delegate: delegate, delegateQueue: queue) + defer { session.finishTasksAndInvalidate() } + // Created up front: cancelling before it exists would invalidate the session, and creating it then throws. + let task = session.dataTask(with: url) + try await withTaskCancellationHandler { + try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in + queue.addOperation { + if let outcome = delegate.outcome { + continuation.resume(with: outcome) + } else { + delegate.continuation = continuation + task.resume() + } + } + } + } onCancel: { + task.cancel() + } + } + + func urlSession(_ session: URLSession, dataTask: URLSessionDataTask, didReceive response: URLResponse, + completionHandler: @escaping @Sendable (URLSession.ResponseDisposition) -> Void) { + if let http = response as? HTTPURLResponse, !(200..<300).contains(http.statusCode) { + writeError = URLError(.badServerResponse) + return completionHandler(.cancel) + } + do { + guard FileManager.default.createFile(atPath: destination.path, contents: nil) else { + throw CocoaError(.fileWriteUnknown, userInfo: [NSFilePathErrorKey: destination.path]) + } + handle = try FileHandle(forWritingTo: destination) + } catch { + writeError = error + return completionHandler(.cancel) + } + if response.expectedContentLength > 0 { expected = response.expectedContentLength } + completionHandler(.allow) + } + + func urlSession(_ session: URLSession, dataTask: URLSessionDataTask, didReceive data: Data) { + guard let handle, writeError == nil else { return } + do { + try handle.write(contentsOf: data) + written += Int64(data.count) + progress(written, expected) + } catch { + writeError = error + dataTask.cancel() + } + } + + func urlSession(_ session: URLSession, task: URLSessionTask, didCompleteWithError error: (any Error)?) { + try? handle?.close() + handle = nil + // Ours comes first: cancelling after it adds a "cancelled" error. + let result: Result = if let failure = writeError ?? error { .failure(failure) } else { .success(()) } + if let waiting = continuation { + continuation = nil + waiting.resume(with: result) + } else { + outcome = result + } + } +} + +extension FileDownload { + static func removeFolders(prefix: String) { + let temporary = URL(fileURLWithPath: NSTemporaryDirectory()) + let names = (try? FileManager.default.contentsOfDirectory(atPath: temporary.path)) ?? [] + for name in names where name.hasPrefix(prefix) { + try? FileManager.default.removeItem(at: temporary.appendingPathComponent(name)) + } + } +} diff --git a/Sources/KnotchyCore/Setup/RuntimeInstaller.swift b/Sources/KnotchyCore/Setup/RuntimeInstaller.swift index f1c893c..8137c43 100644 --- a/Sources/KnotchyCore/Setup/RuntimeInstaller.swift +++ b/Sources/KnotchyCore/Setup/RuntimeInstaller.swift @@ -58,7 +58,7 @@ public struct SystemRuntimeInstaller: RuntimeInstalling { } let package = DownloadedPackage(directory: directory, name: release.url.lastPathComponent) do { - try await PackageDownload.fetch(release.url, to: package.file, progress: progress) + try await FileDownload.fetch(release.url, to: package.file) { written, _ in progress(written) } } catch { discard(package) throw DownloadError.failed(error.localizedDescription) @@ -76,11 +76,7 @@ public struct SystemRuntimeInstaller: RuntimeInstalling { /// Folders left by quitting mid-download or a crash. public static func removeLeftovers() { - let temporary = URL(fileURLWithPath: NSTemporaryDirectory()) - let names = (try? FileManager.default.contentsOfDirectory(atPath: temporary.path)) ?? [] - for name in names where name.hasPrefix("knotchy-runtime-") { - try? FileManager.default.removeItem(at: temporary.appendingPathComponent(name)) - } + FileDownload.removeFolders(prefix: "knotchy-runtime-") } @MainActor @@ -122,88 +118,3 @@ private final class RunningInstaller: InstallerSession { var hasQuit: Bool { app.isTerminated } } - -/// A data task streamed into the package file, so the partial download lives only in its own folder. -private final class PackageDownload: NSObject, URLSessionDataDelegate, @unchecked Sendable { - // Touched only on the session's serial delegate queue. - private var continuation: CheckedContinuation? - /// A completion that arrived before anyone waited: a task cancelled before it started. - private var outcome: Result? - private var handle: FileHandle? - private var written: Int64 = 0 - private var writeError: (any Error)? - private let destination: URL - private let progress: @Sendable (Int64) -> Void - - private init(destination: URL, progress: @escaping @Sendable (Int64) -> Void) { - self.destination = destination - self.progress = progress - } - - static func fetch(_ url: URL, to destination: URL, progress: @escaping @Sendable (Int64) -> Void) async throws { - let delegate = PackageDownload(destination: destination, progress: progress) - let queue = OperationQueue() - queue.maxConcurrentOperationCount = 1 - let session = URLSession(configuration: .ephemeral, delegate: delegate, delegateQueue: queue) - defer { session.finishTasksAndInvalidate() } - // Created up front: cancelling before it exists would invalidate the session, and creating it then throws. - let task = session.dataTask(with: url) - try await withTaskCancellationHandler { - try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in - queue.addOperation { - if let outcome = delegate.outcome { - continuation.resume(with: outcome) - } else { - delegate.continuation = continuation - task.resume() - } - } - } - } onCancel: { - task.cancel() - } - } - - func urlSession(_ session: URLSession, dataTask: URLSessionDataTask, didReceive response: URLResponse, - completionHandler: @escaping @Sendable (URLSession.ResponseDisposition) -> Void) { - if let http = response as? HTTPURLResponse, !(200..<300).contains(http.statusCode) { - writeError = URLError(.badServerResponse) - return completionHandler(.cancel) - } - do { - guard FileManager.default.createFile(atPath: destination.path, contents: nil) else { - throw CocoaError(.fileWriteUnknown, userInfo: [NSFilePathErrorKey: destination.path]) - } - handle = try FileHandle(forWritingTo: destination) - } catch { - writeError = error - return completionHandler(.cancel) - } - completionHandler(.allow) - } - - func urlSession(_ session: URLSession, dataTask: URLSessionDataTask, didReceive data: Data) { - guard let handle, writeError == nil else { return } - do { - try handle.write(contentsOf: data) - written += Int64(data.count) - progress(written) - } catch { - writeError = error - dataTask.cancel() - } - } - - func urlSession(_ session: URLSession, task: URLSessionTask, didCompleteWithError error: (any Error)?) { - try? handle?.close() - handle = nil - // Ours comes first: cancelling after it adds a "cancelled" error. - let result: Result = if let failure = writeError ?? error { .failure(failure) } else { .success(()) } - if let waiting = continuation { - continuation = nil - waiting.resume(with: result) - } else { - outcome = result - } - } -} diff --git a/Sources/KnotchyCore/Setup/SetupFlow.swift b/Sources/KnotchyCore/Setup/SetupFlow.swift index 6cb7603..2119f32 100644 --- a/Sources/KnotchyCore/Setup/SetupFlow.swift +++ b/Sources/KnotchyCore/Setup/SetupFlow.swift @@ -17,8 +17,10 @@ public final class SetupFlow { case update } - public static let requiredFreeBytes: Int64 = 3_000_000_000 + /// The build peaks at about 9 GB: the builder alone takes 4 GB until it is deleted. About 5 GB stays in use. + public static let requiredFreeBytes: Int64 = 10_000_000_000 static let vmDetail = "About 3 minutes · downloads about 225 MB" + static let kernelDownloading = "Downloading the Linux kernel…" public private(set) var mac: Status = .pending("Checking…") public private(set) var unsupported: Unsupported? @@ -51,10 +53,11 @@ public final class SetupFlow { /// With the controller's live progress on top, so a setup the notch started shows here too. public var runtime: Status { - if case .settingUp(let issue) = driver.state, issue == .serviceStopped || issue == .kernelMissing { - return .running("Starting the runtime…", fraction: nil) + switch driver.state { + case .settingUp(.serviceStopped): return .running("Starting the runtime…", fraction: nil) + case .settingUp(.kernelMissing): return .running(driver.setupLog ?? Self.kernelDownloading, fraction: nil) + default: return runtimeBase } - return runtimeBase } public var vm: Status { @@ -142,6 +145,9 @@ public final class SetupFlow { switch await driver.checkSetup() { case .ready?: vmBase = .done("Ready") case .needs(let issue)? where Self.isImageIssue(issue): vmBase = .pending(Self.vmDetail) + case .needs(.kernelMissing)?: + runtimeBase = .pending(Self.kernelDetail(version)) + vmBase = .pending(Self.vmDetail) case .failed(let issue, let message)?: fail(issue, message) case nil where Self.isBrowserOpen(driver.state): vmBase = .done("Ready") // A running VM has an image. case .needs?, nil: break // Busy, or the runtime row's business: the live rows show it. @@ -264,7 +270,7 @@ public final class SetupFlow { continue } switch checked { - case .needs(let issue) where Self.isImageIssue(issue): + case .needs(let issue) where Self.isImageIssue(issue) || issue == .kernelMissing: guard let result = await driver.runSetup(issue, continuing: true) else { continue } return apply(result) case .needs: @@ -279,14 +285,25 @@ public final class SetupFlow { private func apply(_ outcome: SetupRunOutcome) { switch outcome { case .ready: + kernelInstalled() vmBase = .done("Ready") case .needs(let issue): - if Self.isImageIssue(issue) { vmBase = .pending(Self.vmDetail) } + if Self.isImageIssue(issue) { + kernelInstalled() + vmBase = .pending(Self.vmDetail) + } case .failed(let issue, let message): + if Self.isImageIssue(issue) { kernelInstalled() } fail(issue, message) } } + /// The run got past the kernel, so a row still waiting for it is done. + private func kernelInstalled() { + guard case .pending = runtimeBase, runtimeAction == nil, case .current(let version)? = runtimeStatus else { return } + runtimeBase = .done(Self.installedText(version)) + } + private func fail(_ issue: SetupIssue, _ message: String) { if Self.isImageIssue(issue) { vmBase = .failed(message, details: driver.setupOutput) @@ -309,7 +326,7 @@ public final class SetupFlow { guard let facts else { return } let free = facts.bytesFree.map { "\(Self.gigabytes($0)) GB free" } if needsSpace, let bytes = facts.bytesFree, bytes < Self.requiredFreeBytes { - mac = .failed("Setting up needs about 3 GB free. \(Self.gigabytes(bytes)) GB is free.", details: []) + mac = .failed("Setting up needs about 10 GB free. \(Self.gigabytes(bytes)) GB is free.", details: []) } else { mac = .done([facts.osVersion, free].compactMap { $0 }.joined(separator: " · ")) } @@ -325,13 +342,17 @@ public final class SetupFlow { // MARK: - Text static func installDetail(_ release: ContainerRuntime.Release) -> String { - "Apple's container \(release.version) · \(megabytes(release.size)) MB · installs with Apple's Installer" + "Apple's container \(release.version) · \(megabytes(release.size)) MB, then a 700 MB Linux kernel · installs with Apple's Installer" } static func installedText(_ version: String?) -> String { version.map { "container \($0) installed" } ?? "container installed" } + static func kernelDetail(_ version: String?) -> String { + "\(installedText(version)) · still needs its Linux kernel (700 MB)" + } + static func downloadText(_ bytes: Int64, of total: Int64) -> String { "Downloading… \(megabytes(bytes)) of \(megabytes(total)) MB" } diff --git a/Tests/KnotchyCoreTests/ContainerCLIBackendSetupTests.swift b/Tests/KnotchyCoreTests/ContainerCLIBackendSetupTests.swift new file mode 100644 index 0000000..5030fa8 --- /dev/null +++ b/Tests/KnotchyCoreTests/ContainerCLIBackendSetupTests.swift @@ -0,0 +1,163 @@ +import Foundation +import Synchronization +import Testing +@testable import KnotchyCore + +/// Runs `fix` against a fake `container` that logs its arguments. +struct ContainerCLIBackendSetupTests { + private struct FakeCLI { + let directory: URL + let runner: CLIRunner + var log: [String] { + let text = (try? String(contentsOf: directory.appendingPathComponent("log"), encoding: .utf8)) ?? "" + return text.split(separator: "\n").map(String.init) + } + } + + /// `properties` is what `system property list --format json` prints. + private func fakeCLI(properties: String = "{}") throws -> FakeCLI { + let directory = FileManager.default.temporaryDirectory.appendingPathComponent("cli-\(UUID().uuidString)") + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + try properties.write(to: directory.appendingPathComponent("properties.json"), atomically: true, encoding: .utf8) + let tool = directory.appendingPathComponent("container") + let script = """ + #!/bin/sh + echo "$*" >> '\(directory.path)/log' + # A test makes a command fail by creating fail- or fail--. + if [ -f '\(directory.path)/fail-'"$1" ] || [ -f '\(directory.path)/fail-'"$1-$2" ]; then exit 1; fi + if [ "$1 $2 $3" = "system property list" ]; then cat '\(directory.path)/properties.json'; exit 0; fi + if [ "$1 $2" = "builder status" ]; then cat '\(directory.path)/builder.json' 2>/dev/null; exit 0; fi + # The archive must still be there when it's installed. + if [ "$1 $2 $3 $4" = "system kernel set --tar" ]; then [ -f "$5" ] || exit 3; fi + exit 0 + + """ + try script.write(to: tool, atomically: true, encoding: .utf8) + try FileManager.default.setAttributes([.posixPermissions: 0o755], ofItemAtPath: tool.path) + return FakeCLI(directory: directory, runner: CLIRunner(searchPaths: [tool.path])) + } + + private func fail(_ command: String, in cli: FakeCLI) throws { + try Data().write(to: cli.directory.appendingPathComponent("fail-\(command)")) + } + + /// What `builder status --format json` prints; without it the state can't be read. + private func builder(_ json: String, in cli: FakeCLI) throws { + try json.write(to: cli.directory.appendingPathComponent("builder.json"), atomically: true, encoding: .utf8) + } + + private func build(with cli: FakeCLI) async throws { + try await ContainerCLIBackend(vmSources: cli.directory, cli: cli.runner).fix(.imageMissing) { _ in } + } + + private func kernelProperties(url: URL) -> String { + #"{"kernel":{"binaryPath":"opt/kata/vmlinux","digest":"sha256:abc123","url":"\#(url.absoluteString)"}}"# + } + + @Test func startingTheServiceLeavesTheKernelToItsOwnStep() async throws { + let cli = try fakeCLI() + defer { try? FileManager.default.removeItem(at: cli.directory) } + try await ContainerCLIBackend(vmSources: nil, cli: cli.runner).fix(.serviceStopped) { _ in } + #expect(cli.log == ["system start --disable-kernel-install"]) + } + + @Test func theKernelIsDownloadedWithProgressThenInstalledFromTheFile() async throws { + let archive = FileManager.default.temporaryDirectory.appendingPathComponent("kata-\(UUID().uuidString).tar.zst") + try Data(count: 3_000_000).write(to: archive) + defer { try? FileManager.default.removeItem(at: archive) } + let cli = try fakeCLI(properties: kernelProperties(url: archive)) + defer { try? FileManager.default.removeItem(at: cli.directory) } + let lines = LineLog() + + try await ContainerCLIBackend(vmSources: nil, cli: cli.runner).fix(.kernelMissing) { lines.append($0) } + + #expect(cli.log.count == 2) + #expect(cli.log.first == "system property list --format json") + let install = try #require(cli.log.last?.split(separator: " ").map(String.init)) + #expect(Array(install.prefix(4)) == ["system", "kernel", "set", "--tar"]) + #expect(Array(install.dropFirst(5)) == ["--binary", "opt/kata/vmlinux", "--digest", "sha256:abc123", "--force"]) + // The download is deleted once the runtime has its copy. + let downloaded = URL(fileURLWithPath: install[4]) + #expect(downloaded.deletingLastPathComponent().lastPathComponent.hasPrefix("knotchy-kernel-")) + #expect(!FileManager.default.fileExists(atPath: downloaded.deletingLastPathComponent().path)) + #expect(lines.all.contains("Downloading the Linux kernel… 3 of 3 MB")) + #expect(lines.all.last == "Installing the Linux kernel…") + } + + @Test func aFailedKernelDownloadInstallsNothing() async throws { + let missing = FileManager.default.temporaryDirectory.appendingPathComponent("missing-\(UUID().uuidString).tar.zst") + let cli = try fakeCLI(properties: kernelProperties(url: missing)) + defer { try? FileManager.default.removeItem(at: cli.directory) } + await #expect(throws: VMBackendError.failed("Couldn't download the Linux kernel. Check your connection and try again.")) { + try await ContainerCLIBackend(vmSources: nil, cli: cli.runner).fix(.kernelMissing) { _ in } + } + #expect(cli.log == ["system property list --format json"]) + } + + @Test func aRebuildDeletesTheImageItReplaced() async throws { + let cli = try fakeCLI() + defer { try? FileManager.default.removeItem(at: cli.directory) } + try await build(with: cli) + #expect(cli.log.count == 4) + #expect(cli.log.first == "builder status --format json") + #expect(cli.log.dropFirst().first == "image tag local/knotchy-vm:latest local/knotchy-vm:previous") + #expect(cli.log.dropFirst(2).first?.hasPrefix("build ") == true) + #expect(cli.log.last == "image delete local/knotchy-vm:previous") + } + + @Test func aFirstBuildHasNothingToDelete() async throws { + let cli = try fakeCLI() + defer { try? FileManager.default.removeItem(at: cli.directory) } + try fail("image-tag", in: cli) + try await build(with: cli) + #expect(!cli.log.contains { $0.hasPrefix("image delete") }) + #expect(cli.log.last?.hasPrefix("build ") == true) + } + + @Test func aFailedBuildKeepsTheImageButDropsTheExtraNameAndTheBuilder() async throws { + let cli = try fakeCLI() + defer { try? FileManager.default.removeItem(at: cli.directory) } + try builder("[]", in: cli) + try fail("build", in: cli) + await #expect(throws: VMBackendError.self) { try await build(with: cli) } + #expect(Array(cli.log.suffix(2)) == ["image delete local/knotchy-vm:previous", "builder delete --force"]) + #expect(!cli.log.contains { $0.hasPrefix("image delete local/knotchy-vm:latest") }) + } + + @Test func aBuilderTheBuildCreatedIsDeletedAfterwards() async throws { + let cli = try fakeCLI() + defer { try? FileManager.default.removeItem(at: cli.directory) } + try builder("[]", in: cli) + try await build(with: cli) + #expect(cli.log.last == "builder delete --force") + } + + @Test func aStoppedBuilderIsStoppedAgain() async throws { + let cli = try fakeCLI() + defer { try? FileManager.default.removeItem(at: cli.directory) } + try builder(#"[{"id":"buildkit","status":{"state":"stopped"}}]"#, in: cli) + try await build(with: cli) + #expect(cli.log.last == "builder stop") + } + + @Test func aRunningBuilderIsLeftRunning() async throws { + let cli = try fakeCLI() + defer { try? FileManager.default.removeItem(at: cli.directory) } + try builder(#"[{"id":"buildkit","status":{"state":"running"}}]"#, in: cli) + try await build(with: cli) + #expect(cli.log.filter { $0.hasPrefix("builder") } == ["builder status --format json"]) + } + + @Test func aRuntimeThatDoesntNameItsKernelInstallsItsOwnRecommendation() async throws { + let cli = try fakeCLI(properties: #"{"kernel":{}}"#) + defer { try? FileManager.default.removeItem(at: cli.directory) } + try await ContainerCLIBackend(vmSources: nil, cli: cli.runner).fix(.kernelMissing) { _ in } + #expect(cli.log == ["system property list --format json", "system kernel set --recommended --force"]) + } +} + +private final class LineLog: Sendable { + private let lines = Mutex<[String]>([]) + var all: [String] { lines.withLock { $0 } } + func append(_ line: String) { lines.withLock { $0.append(line) } } +} diff --git a/Tests/KnotchyCoreTests/SessionSetupRunTests.swift b/Tests/KnotchyCoreTests/SessionSetupRunTests.swift index f7b1782..c08e02c 100644 --- a/Tests/KnotchyCoreTests/SessionSetupRunTests.swift +++ b/Tests/KnotchyCoreTests/SessionSetupRunTests.swift @@ -13,6 +13,22 @@ struct SessionSetupRunTests { #expect(h.backend.calls == [.sweep(keeping: []), .fix(.serviceStopped), .preflight, .fix(.imageMissing), .preflight]) } + @Test func aWindowRunFetchesTheKernelBetweenTheServiceAndTheBuild() async { + let h = Harness() + h.backend.queuePreflight(.issue(.kernelMissing), .issue(.imageMissing)) + #expect(await h.controller.runSetup(.serviceStopped, continuing: true) == .ready) + #expect(h.backend.calls == [.sweep(keeping: []), .fix(.serviceStopped), .preflight, .fix(.kernelMissing), .preflight, + .fix(.imageMissing), .preflight]) + } + + @Test func aKernelThatStaysMissingIsReportedOnce() async { + let h = Harness() + h.backend.queuePreflight(.issue(.kernelMissing)) + #expect(await h.controller.runSetup(.kernelMissing, continuing: true) + == .failed(.kernelMissing, SessionController.stillBrokenMessage)) + #expect(h.backend.calls.filter { $0 == .fix(.kernelMissing) }.count == 1) + } + @Test func aWindowRunStartsFolded() async { let h = Harness() h.backend.hold(.fix) @@ -79,6 +95,22 @@ struct SessionSetupRunTests { #expect(h.backend.calls == [.sweep(keeping: []), .preflight, .fix(.serviceStopped), .preflight]) } + @Test func checkingNeverDownloadsTheKernel() async { + let h = Harness() + h.backend.queuePreflight(.issue(.serviceStopped), .issue(.kernelMissing)) + #expect(await h.controller.checkSetup() == .needs(.kernelMissing)) + #expect(h.state == .idle) + #expect(h.backend.calls == [.sweep(keeping: []), .preflight, .fix(.serviceStopped), .preflight]) + } + + @Test func setupErrorsReadAsSentences() { + let timedOut = SessionController.message(for: CLIRunner.Failure.timedOut(["system", "start"])) + #expect(timedOut == "The container runtime stopped responding. Try again.") + #expect(SessionController.message(for: CLIRunner.Failure.notInstalled) == "Apple's container runtime isn't installed.") + #expect(SessionController.message(for: CLIRunner.Failure.launchFailed("denied")) + == "Couldn't run the container runtime: denied") + } + @Test func checkingOpensALinkACardWasHolding() async { let h = Harness() h.backend.queuePreflight(.issue(.imageMissing)) diff --git a/Tests/KnotchyCoreTests/SessionSetupTests.swift b/Tests/KnotchyCoreTests/SessionSetupTests.swift index 0bc8caa..2e06fd2 100644 --- a/Tests/KnotchyCoreTests/SessionSetupTests.swift +++ b/Tests/KnotchyCoreTests/SessionSetupTests.swift @@ -270,6 +270,15 @@ struct SessionSetupTests { .start("knotchy-00000002")]) } + @Test func aLinkFetchesAMissingKernelAfterStartingTheService() async { + let h = Harness() + h.backend.queuePreflight(.issue(.serviceStopped), .issue(.kernelMissing)) + h.controller.open(Harness.url) + await waitUntil("loading") { h.state == .starting(.loading, folded: false) } + #expect(h.backend.calls == [.sweep(keeping: []), .preflight, .fix(.serviceStopped), .preflight, + .fix(.kernelMissing), .preflight, .start("knotchy-00000003")]) + } + @Test func aServiceThatWontStayUpShowsItsCardOnce() async { let h = Harness() h.backend.queuePreflight(.issue(.serviceStopped), .issue(.serviceStopped)) diff --git a/Tests/KnotchyCoreTests/SetupFlowTests.swift b/Tests/KnotchyCoreTests/SetupFlowTests.swift index 06cc12e..a3ff846 100644 --- a/Tests/KnotchyCoreTests/SetupFlowTests.swift +++ b/Tests/KnotchyCoreTests/SetupFlowTests.swift @@ -28,7 +28,7 @@ struct SetupFlowTests { let flow = makeFlow() await flow.refresh() #expect(flow.runtimeAction == .install) - #expect(flow.runtime == .pending("Apple's container 1.4.1 · 118 MB · installs with Apple's Installer")) + #expect(flow.runtime == .pending("Apple's container 1.4.1 · 118 MB, then a 700 MB Linux kernel · installs with Apple's Installer")) #expect(flow.vm == .pending("About 3 minutes · downloads about 225 MB")) #expect(flow.canSetUp) #expect(driver.calls == ["runtimeStatus"]) @@ -56,7 +56,7 @@ struct SetupFlowTests { driver.runtimeStatuses = [.missing] let flow = makeFlow() await flow.refresh() - #expect(flow.mac == .failed("Setting up needs about 3 GB free. 2.5 GB is free.", details: [])) + #expect(flow.mac == .failed("Setting up needs about 10 GB free. 2.5 GB is free.", details: [])) #expect(!flow.canSetUp) driver.runtimeStatuses = [.current("1.4.1")] @@ -190,6 +190,36 @@ struct SetupFlowTests { #expect(!driver.calls.contains { $0.hasPrefix("runSetup") }) } + @Test func aMissingKernelWaitsForSetUp() async { + driver.checkResults = [.needs(.kernelMissing)] + let flow = makeFlow() + await flow.refresh() + #expect(flow.runtime == .pending("container 1.4.1 installed · still needs its Linux kernel (700 MB)")) + #expect(flow.vm == .pending("About 3 minutes · downloads about 225 MB")) + #expect(flow.canSetUp) + #expect(!driver.calls.contains { $0.hasPrefix("runSetup") }) + } + + @Test func setUpFetchesAMissingKernelThenBuilds() async { + driver.checkResults = [.needs(.kernelMissing), .needs(.kernelMissing)] + let flow = makeFlow() + await flow.setUp() + #expect(driver.calls.contains("runSetup(kernelMissing, continuing: true)")) + #expect(flow.runtime == .done("container 1.4.1 installed")) + #expect(flow.vm == .done("Ready")) + } + + @Test func aFailedKernelDownloadShowsInTheRuntimeRow() async { + driver.checkResults = [.needs(.kernelMissing), .needs(.kernelMissing)] + driver.runResults = [.failed(.kernelMissing, "Couldn't download the Linux kernel. Check your connection and try again.")] + driver.setupOutput = ["Downloading the Linux kernel… 212 of 697 MB"] + let flow = makeFlow() + await flow.setUp() + #expect(flow.runtime == .failed("Couldn't download the Linux kernel. Check your connection and try again.", + details: ["Downloading the Linux kernel… 212 of 697 MB"])) + #expect(flow.vm == .pending("About 3 minutes · downloads about 225 MB")) + } + @Test func aBuildFailureShowsItsOutput() async { driver.checkResults = [.needs(.imageMissing), .needs(.imageMissing)] driver.runResults = [.failed(.imageMissing, "The image build failed: out of space")] @@ -215,7 +245,12 @@ struct SetupFlowTests { driver.setupLog = "Step 3/9" #expect(flow.vm == .running("Step 3/9", fraction: nil)) driver.state = .settingUp(.serviceStopped) + driver.setupLog = nil #expect(flow.runtime == .running("Starting the runtime…", fraction: nil)) + driver.state = .settingUp(.kernelMissing) + #expect(flow.runtime == .running("Downloading the Linux kernel…", fraction: nil)) + driver.setupLog = "Downloading the Linux kernel… 212 of 697 MB" + #expect(flow.runtime == .running("Downloading the Linux kernel… 212 of 697 MB", fraction: nil)) } @Test func anOpenBrowserCountsAsAReadyVM() async {