Initial release

Sources/KnotchyCore/Stream/ContentReadiness.swift

@@ -0,0 +1,39 @@
+/// Decides when Starting ends. Frames and progress before NAVIGATED come from the agent's
+/// about:blank bootstrap and never count; `afterHello` is a backstop if NAVIGATED never arrives.
+public struct ContentReadiness: Sendable {
+    public static let afterNavigated: Duration = .seconds(15)
+    public static let afterHello: Duration = .seconds(25)
+
+    public private(set) var isReady = false
+    private var helloAt: ContinuousClock.Instant?
+    private var navigatedAt: ContinuousClock.Instant?
+
+    public init() {}
+
+    public mutating func hello(at now: ContinuousClock.Instant) {
+        if helloAt == nil { helloAt = now }
+    }
+
+    public mutating func navigated(at now: ContinuousClock.Instant) {
+        if navigatedAt == nil { navigatedAt = now }
+    }
+
+    public mutating func frame(isUniform: Bool) {
+        if navigatedAt != nil, !isUniform { isReady = true }
+    }
+
+    public mutating func progress(_ percent: Int) {
+        if navigatedAt != nil, percent >= 100 { isReady = true }
+    }
+
+    public mutating func tick(now: ContinuousClock.Instant) {
+        if let deadline = nextDeadline, now >= deadline { isReady = true }
+    }
+
+    public var nextDeadline: ContinuousClock.Instant? {
+        guard !isReady else { return nil }
+        return [navigatedAt.map { $0 + Self.afterNavigated }, helloAt.map { $0 + Self.afterHello }]
+            .compactMap { $0 }
+            .min()
+    }
+}

Sources/KnotchyCore/Stream/FrameSampler.swift

@@ -0,0 +1,22 @@
+/// Tells a blank frame from one with page content.
+public enum FrameSampler {
+    public static let grid = 48
+    public static let maxDimension = 1 << 15
+    public static let maxBytesPerRow = 1 << 20
+
+    /// Malformed input counts as uniform so it can never pose as content; the caps prevent overflow.
+    public static func isUniform(_ pixels: UnsafeRawBufferPointer, width: Int, height: Int, bytesPerRow: Int) -> Bool {
+        guard (1...maxDimension).contains(width), (1...maxDimension).contains(height),
+              bytesPerRow >= width * 4, bytesPerRow <= maxBytesPerRow,
+              pixels.count >= (height - 1) * bytesPerRow + width * 4
+        else { return true }
+        let first = pixels.loadUnaligned(fromByteOffset: 0, as: UInt32.self)
+        for gy in 0..<grid {
+            for gx in 0..<grid {
+                let offset = (gy * height / grid) * bytesPerRow + (gx * width / grid) * 4
+                if pixels.loadUnaligned(fromByteOffset: offset, as: UInt32.self) != first { return false }
+            }
+        }
+        return true
+    }
+}

Sources/KnotchyCore/Stream/H264.swift

@@ -0,0 +1,88 @@
+import Foundation
+
+public enum NALUnitType {
+    public static let slice: UInt8 = 1
+    public static let idrSlice: UInt8 = 5
+    public static let sei: UInt8 = 6
+    public static let sps: UInt8 = 7
+    public static let pps: UInt8 = 8
+    public static let accessUnitDelimiter: UInt8 = 9
+}
+
+public enum AnnexB {
+    /// Splits an Annex-B stream into NAL units. Trailing zeros are trimmed: a NAL unit never ends in 0x00.
+    public static func nalUnits(in stream: Data) -> [Data] {
+        let bytes = [UInt8](stream)
+        var starts: [(code: Int, payload: Int)] = []
+        var i = 0
+        while i + 2 < bytes.count {
+            if bytes[i] == 0, bytes[i + 1] == 0, bytes[i + 2] == 1 {
+                starts.append((code: i, payload: i + 3))
+                i += 3
+            } else {
+                i += 1
+            }
+        }
+        return starts.indices.compactMap { k in
+            var end = k + 1 < starts.count ? starts[k + 1].code : bytes.count
+            while end > starts[k].payload, bytes[end - 1] == 0 { end -= 1 }
+            return end > starts[k].payload ? Data(bytes[starts[k].payload..<end]) : nil
+        }
+    }
+}
+
+public struct ParameterSets: Equatable, Sendable {
+    public var sps: Data
+    public var pps: Data
+
+    public init(sps: Data, pps: Data) {
+        self.sps = sps
+        self.pps = pps
+    }
+}
+
+public struct ParsedAccessUnit: Equatable, Sendable {
+    /// AVCC-framed slice NAL units; empty if the access unit has none, so don't decode it.
+    public var sample = Data()
+    public var isIDR = false
+    /// Set only on the unit that changes the SPS/PPS; if that unit is dropped, check `H264Parser.parameterSets`.
+    public var newParameterSets: ParameterSets?
+}
+
+public struct H264Parser: Sendable {
+    public private(set) var parameterSets: ParameterSets?
+
+    public init() {}
+
+    public mutating func parse(_ accessUnit: Data) -> ParsedAccessUnit {
+        var unit = ParsedAccessUnit()
+        var sps: Data?
+        var pps: Data?
+        for nal in AnnexB.nalUnits(in: accessUnit) {
+            let header = nal[nal.startIndex]
+            guard header & 0x80 == 0 else { continue }   // forbidden_zero_bit
+            let type = header & 0x1F
+            switch type {
+            case NALUnitType.sps:
+                sps = nal
+            case NALUnitType.pps:
+                pps = nal
+            case NALUnitType.slice, NALUnitType.idrSlice:
+                if type == NALUnitType.idrSlice { unit.isIDR = true }
+                unit.sample.appendBE32(UInt32(nal.count))
+                unit.sample.append(nal)
+            default:
+                break
+            }
+        }
+        // SPS and PPS must come from the same access unit; x264 repeats both on every IDR.
+        if let sps, let pps {
+            let sets = ParameterSets(sps: sps, pps: pps)
+            if sets != parameterSets {
+                parameterSets = sets
+                unit.newParameterSets = sets
+            }
+        }
+        return unit
+    }
+}

Sources/KnotchyCore/Stream/InputEvent.swift

@@ -0,0 +1,78 @@
+import Foundation
+
+public struct Modifiers: OptionSet, Hashable, Sendable {
+    public let rawValue: UInt8
+
+    public init(rawValue: UInt8) {
+        self.rawValue = rawValue
+    }
+
+    public static let shift = Modifiers(rawValue: 1 << 0)
+    public static let control = Modifiers(rawValue: 1 << 1)
+    public static let alt = Modifiers(rawValue: 1 << 2)
+    public static let meta = Modifiers(rawValue: 1 << 3)
+    public static let leftButton = Modifiers(rawValue: 1 << 4)
+    public static let rightButton = Modifiers(rawValue: 1 << 5)
+    public static let middleButton = Modifiers(rawValue: 1 << 6)
+}
+
+public enum PointerButton: UInt8, Sendable {
+    case left = 1
+    case right = 2
+    case middle = 3
+}
+
+/// A viewer → agent message. Coordinates are render pixels from the top left; negative scroll is down/right.
+public enum InputEvent: Equatable, Sendable {
+    case pointerMove(x: UInt16, y: UInt16, mods: Modifiers)
+    case pointerButton(PointerButton, pressed: Bool, x: UInt16, y: UInt16, mods: Modifiers)
+    case scroll(ticksX: Int16, ticksY: Int16, x: UInt16, y: UInt16)
+    case key(pressed: Bool, mods: Modifiers, name: String)
+    case navigate(String)
+    case script(String)
+    case keyframe
+
+    /// Nil if the agent can't use it: a payload over 64 KiB, an empty KEY name, or a NUL in a string.
+    public func encoded() -> Data? {
+        var payload = Data()
+        let type: UInt8
+        switch self {
+        case .pointerMove(let x, let y, let mods):
+            type = MessageType.pointerMove
+            payload.appendBE16(x)
+            payload.appendBE16(y)
+            payload.append(mods.rawValue)
+        case .pointerButton(let button, let pressed, let x, let y, let mods):
+            type = MessageType.pointerButton
+            payload.append(button.rawValue)
+            payload.append(pressed ? 1 : 0)
+            payload.appendBE16(x)
+            payload.appendBE16(y)
+            payload.append(mods.rawValue)
+        case .scroll(let ticksX, let ticksY, let x, let y):
+            type = MessageType.scroll
+            payload.appendBE16(UInt16(bitPattern: ticksX))
+            payload.appendBE16(UInt16(bitPattern: ticksY))
+            payload.appendBE16(x)
+            payload.appendBE16(y)
+        case .key(let pressed, let mods, let name):
+            guard !name.isEmpty, !name.utf8.contains(0) else { return nil }
+            type = MessageType.key
+            payload.append(pressed ? 1 : 0)
+            payload.append(mods.rawValue)
+            payload.append(contentsOf: Array(name.utf8))
+        case .navigate(let url):
+            guard !url.utf8.contains(0) else { return nil }
+            type = MessageType.navigate
+            payload.append(contentsOf: Array(url.utf8))
+        case .script(let javaScript):
+            guard !javaScript.utf8.contains(0) else { return nil }
+            type = MessageType.script
+            payload.append(contentsOf: Array(javaScript.utf8))
+        case .keyframe:
+            type = MessageType.keyframe
+        }
+        guard payload.count <= WireProtocol.maxOtherPayload else { return nil }
+        return framed(type, payload)
+    }
+}

Sources/KnotchyCore/Stream/StreamClient.swift

@@ -0,0 +1,348 @@
+import CoreVideo
+import Darwin
+import Foundation
+
+public struct StreamClient: StreamConnector {
+    public let renderSize: PixelSize
+
+    public init(renderSize: PixelSize) {
+        self.renderSize = renderSize
+    }
+
+    public func connect(socketPath: String) async throws -> any StreamConnection {
+        let renderSize = renderSize
+        return try await withCheckedThrowingContinuation { continuation in
+            DispatchQueue.global(qos: .userInitiated).async {
+                continuation.resume(with: Result { try Self.connectBlocking(path: socketPath, renderSize: renderSize) })
+            }
+        }
+    }
+
+    static func connectBlocking(path: String, renderSize: PixelSize) throws -> StreamSession {
+        let deadline = Date().addingTimeInterval(30)
+        var connectedOnce = false
+        while Date() < deadline {
+            if let fd = openSocket(path) {
+                connectedOnce = true
+                setReadTimeout(fd, seconds: 2)
+                var decoder = AgentMessageDecoder()
+                switch readHello(fd, into: &decoder) {
+                case .hello(let version, let size):
+                    guard version == WireProtocol.version else {
+                        Darwin.close(fd)
+                        throw StreamError.versionMismatch(found: version, need: WireProtocol.version)
+                    }
+                    guard size == renderSize else {
+                        Darwin.close(fd)
+                        throw StreamError.protocolViolation("HELLO is \(size), asked for \(renderSize)")
+                    }
+                    setReadTimeout(fd, seconds: 0)
+                    // A VM that stops reading input must not wedge the session's queue.
+                    setSendTimeout(fd, seconds: 2)
+                    return StreamSession(fd: fd, decoder: decoder, size: size)
+                case .violation(let detail):
+                    Darwin.close(fd)
+                    throw StreamError.protocolViolation(detail)
+                case .none:
+                    Darwin.close(fd)
+                }
+            }
+            usleep(100_000)
+        }
+        throw connectedOnce ? StreamError.timedOut : StreamError.refused
+    }
+
+    private enum HelloOutcome {
+        case hello(version: Int, size: PixelSize)
+        case violation(String)
+        case none
+    }
+
+    private static func readHello(_ fd: Int32, into decoder: inout AgentMessageDecoder) -> HelloOutcome {
+        var buffer = [UInt8](repeating: 0, count: 64 * 1024)
+        while true {
+            let n = buffer.withUnsafeMutableBytes { read(fd, $0.baseAddress!, $0.count) }
+            if n < 0, errno == EINTR { continue }
+            guard n > 0 else { return .none }
+            decoder.append(Data(buffer[0..<n]))
+            do {
+                if case .hello(let version, let size)? = try decoder.next() { return .hello(version: version, size: size) }
+            } catch {
+                return .violation(error.description)
+            }
+        }
+    }
+
+    private static func openSocket(_ path: String) -> Int32? {
+        let fd = socket(AF_UNIX, SOCK_STREAM, 0)
+        guard fd >= 0 else { return nil }
+        var address = sockaddr_un()
+        address.sun_family = sa_family_t(AF_UNIX)
+        let bytes = Array(path.utf8)
+        guard bytes.count < MemoryLayout.size(ofValue: address.sun_path) else {
+            Darwin.close(fd)
+            return nil
+        }
+        withUnsafeMutableBytes(of: &address.sun_path) { $0.copyBytes(from: bytes) }
+        let size = socklen_t(MemoryLayout<sockaddr_un>.size)
+        let result = withUnsafePointer(to: &address) {
+            $0.withMemoryRebound(to: sockaddr.self, capacity: 1) { Darwin.connect(fd, $0, size) }
+        }
+        guard result == 0 else {
+            Darwin.close(fd)
+            return nil
+        }
+        var on: Int32 = 1
+        setsockopt(fd, SOL_SOCKET, SO_NOSIGPIPE, &on, socklen_t(MemoryLayout<Int32>.size))
+        return fd
+    }
+
+    static func setReadTimeout(_ fd: Int32, seconds: Int) {
+        var timeout = timeval.init(tv_sec: seconds, tv_usec: 0)
+        setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &timeout, socklen_t(MemoryLayout<timeval>.size))
+    }
+
+    static func setSendTimeout(_ fd: Int32, seconds: Int) {
+        var timeout = timeval.init(tv_sec: seconds, tv_usec: 0)
+        setsockopt(fd, SOL_SOCKET, SO_SNDTIMEO, &timeout, socklen_t(MemoryLayout<timeval>.size))
+    }
+}
+
+/// Its reader thread must never stop (the VM's watchdog needs a connected viewer); all else runs on `queue`.
+final class StreamSession: StreamConnection, @unchecked Sendable {
+    static let firstFrameTimeout: TimeInterval = 20
+    static let decodeErrorLimit = 10
+
+    let events: AsyncStream<StreamEvent>
+    private let continuation: AsyncStream<StreamEvent>.Continuation
+    private let queue = DispatchQueue(label: "aculix.knotchy.stream")
+
+    private let fd: Int32
+    private var decoder: AgentMessageDecoder
+    private var parser = H264Parser()
+    private var video: VideoDecoder?
+    private var readiness = ContentReadiness()
+    private var announcedReady = false
+    private var enabled = true
+    private var waitingForIDR = false
+    private var frameHandler: (@Sendable (DecodedFrame) -> Void)?
+    private var readinessTimer: DispatchSourceTimer?
+    private var firstFrameTimer: DispatchSourceTimer?
+    private var pendingPageURL: URL?
+    private var pageURLTimer: DispatchSourceTimer?
+    private var decodeErrors: [Date] = []
+    private var ended = false
+
+    init(fd: Int32, decoder: AgentMessageDecoder, size: PixelSize) {
+        self.fd = fd
+        self.decoder = decoder
+        let (stream, continuation) = AsyncStream.makeStream(of: StreamEvent.self)
+        events = stream
+        self.continuation = continuation
+        continuation.yield(.hello(version: WireProtocol.version, size: size))
+        queue.async { self.begin() }
+        let reader = Thread { [self] in readLoop() }
+        reader.name = "knotchy stream reader"
+        reader.start()
+    }
+
+    // MARK: StreamConnection
+
+    var decodeEnabled: Bool {
+        get { queue.sync { enabled } }
+        set {
+            queue.async {
+                if newValue, !self.enabled {
+                    self.waitingForIDR = true
+                    self.write(.keyframe)
+                }
+                self.enabled = newValue
+            }
+        }
+    }
+
+    func setFrameHandler(_ handler: @escaping @Sendable (DecodedFrame) -> Void) {
+        queue.async { self.frameHandler = handler }
+    }
+
+    func send(_ input: InputEvent) {
+        queue.async { self.write(input) }
+    }
+
+    func close() {
+        queue.async { self.end(nil) }
+    }
+
+    // MARK: Reading
+
+    private func readLoop() {
+        var buffer = [UInt8](repeating: 0, count: 64 * 1024)
+        while true {
+            let n = buffer.withUnsafeMutableBytes { read(fd, $0.baseAddress!, $0.count) }
+            if n < 0, errno == EINTR { continue }
+            guard n > 0 else { break }
+            let chunk = Data(buffer[0..<n])
+            queue.async { self.received(chunk) }
+        }
+        queue.async {
+            self.end(.closed)
+            Darwin.close(self.fd)
+        }
+    }
+
+    private func begin() {
+        readiness.hello(at: .now)
+        scheduleReadinessTick()
+        let timer = DispatchSource.makeTimerSource(queue: queue)
+        timer.schedule(deadline: .now() + Self.firstFrameTimeout)
+        timer.setEventHandler { [weak self] in self?.end(.noFirstFrame) }
+        timer.resume()
+        firstFrameTimer = timer
+        received(Data())   // Messages that arrived together with HELLO.
+    }
+
+    private func received(_ chunk: Data) {
+        guard !ended else { return }
+        decoder.append(chunk)
+        do {
+            while let message = try decoder.next() { handle(message) }
+        } catch {
+            end(.protocolViolation(error.description))
+        }
+    }
+
+    private func handle(_ message: AgentMessage) {
+        switch message {
+        case .navigated:
+            readiness.navigated(at: .now)
+            scheduleReadinessTick()
+        case .progress(let percent):
+            readiness.progress(percent)
+            continuation.yield(.progress(percent))
+            announceIfReady()
+        case .video(_, let accessUnit):
+            firstFrameTimer?.cancel()
+            firstFrameTimer = nil
+            let unit = parser.parse(accessUnit)
+            if unit.newParameterSets != nil || video == nil, let sets = parser.parameterSets {
+                video = VideoDecoder(parameterSets: sets) { [weak self] frame, status in
+                    guard let self else { return }
+                    self.queue.async { self.decoded(frame, status: status) }
+                }
+            }
+            guard enabled, !unit.sample.isEmpty, let video else { return }
+            if waitingForIDR {
+                guard unit.isIDR else { return }
+                waitingForIDR = false
+            }
+            video.decode(unit.sample)
+        case .pageURL(let text):
+            guard let url = URL(string: text) else { return }
+            pendingPageURL = url
+            passOnPageURL()
+        case .hello, .unknown:
+            break
+        }
+    }
+
+    private func decoded(_ frame: DecodedFrame?, status: OSStatus) {
+        guard !ended else { return }
+        guard let frame else { return decodeFailed() }
+        if !readiness.isReady {
+            readiness.frame(isUniform: Self.isUniform(frame.pixelBuffer))
+            announceIfReady()
+        }
+        frameHandler?(frame)
+    }
+
+    private func decodeFailed() {
+        let now = Date()
+        decodeErrors = decodeErrors.filter { now.timeIntervalSince($0) < 60 } + [now]
+        if decodeErrors.count > Self.decodeErrorLimit { return end(.decodeErrors) }
+        waitingForIDR = true
+        write(.keyframe)
+    }
+
+    private static func isUniform(_ buffer: CVPixelBuffer) -> Bool {
+        CVPixelBufferLockBaseAddress(buffer, .readOnly)
+        defer { CVPixelBufferUnlockBaseAddress(buffer, .readOnly) }
+        guard let base = CVPixelBufferGetBaseAddress(buffer) else { return true }
+        let bytes = UnsafeRawBufferPointer(start: base, count: CVPixelBufferGetDataSize(buffer))
+        return FrameSampler.isUniform(bytes, width: CVPixelBufferGetWidth(buffer),
+                                      height: CVPixelBufferGetHeight(buffer),
+                                      bytesPerRow: CVPixelBufferGetBytesPerRow(buffer))
+    }
+
+    // MARK: Readiness
+
+    private func announceIfReady() {
+        guard readiness.isReady, !announcedReady else { return }
+        announcedReady = true
+        readinessTimer?.cancel()
+        readinessTimer = nil
+        continuation.yield(.contentReady)
+    }
+
+    private func scheduleReadinessTick() {
+        readinessTimer?.cancel()
+        readinessTimer = nil
+        guard let deadline = readiness.nextDeadline else { return }
+        let wait = deadline - .now
+        let seconds = max(0, Double(wait.components.seconds) + Double(wait.components.attoseconds) / 1e18)
+        let timer = DispatchSource.makeTimerSource(queue: queue)
+        timer.schedule(deadline: .now() + seconds)
+        timer.setEventHandler { [weak self] in
+            guard let self else { return }
+            self.readiness.tick(now: .now)
+            self.announceIfReady()
+        }
+        timer.resume()
+        readinessTimer = timer
+    }
+
+    /// At most 10 a second, always including the last: a page can change its address in a loop.
+    private func passOnPageURL() {
+        guard pageURLTimer == nil, let url = pendingPageURL else { return }
+        pendingPageURL = nil
+        continuation.yield(.pageURL(url))
+        let timer = DispatchSource.makeTimerSource(queue: queue)
+        timer.schedule(deadline: .now() + 0.1)
+        timer.setEventHandler { [weak self] in
+            guard let self, !self.ended else { return }
+            self.pageURLTimer = nil
+            self.passOnPageURL()
+        }
+        timer.resume()
+        pageURLTimer = timer
+    }
+
+    // MARK: Writing and ending
+
+    private func write(_ input: InputEvent) {
+        guard !ended, let data = input.encoded() else { return }
+        let stalled = data.withUnsafeBytes { raw -> Bool in
+            var offset = 0
+            while offset < raw.count {
+                let n = Darwin.write(fd, raw.baseAddress! + offset, raw.count - offset)
+                if n > 0 { offset += n } else if n < 0, errno == EINTR { continue } else {
+                    return n < 0 && (errno == EAGAIN || errno == EWOULDBLOCK)
+                }
+            }
+            return false
+        }
+        if stalled { end(.socketError("The VM stopped taking input.")) }
+    }
+
+    private func end(_ reason: EndReason?) {
+        guard !ended else { return }
+        ended = true
+        shutdown(fd, SHUT_RDWR)
+        readinessTimer?.cancel()
+        firstFrameTimer?.cancel()
+        pageURLTimer?.cancel()
+        video = nil
+        frameHandler = nil
+        if let reason { continuation.yield(.ended(reason)) }
+        continuation.finish()
+    }
+}

Sources/KnotchyCore/Stream/StreamConnection.swift

@@ -0,0 +1,50 @@
+import CoreVideo
+
+public protocol StreamConnector: Sendable {
+    /// Returns once a valid HELLO arrives; a version mismatch is reported before any other HELLO problem.
+    func connect(socketPath: String) async throws -> any StreamConnection
+}
+
+/// `@unchecked`: a decoded buffer is never written after VideoToolbox returns it.
+public struct DecodedFrame: @unchecked Sendable {
+    public let pixelBuffer: CVPixelBuffer
+
+    public init(_ pixelBuffer: CVPixelBuffer) {
+        self.pixelBuffer = pixelBuffer
+    }
+}
+
+public protocol StreamConnection: AnyObject, Sendable {
+    /// Unbounded: reading starts only after `connect` returns, and early events must not be lost.
+    var events: AsyncStream<StreamEvent> { get }
+    var decodeEnabled: Bool { get set }
+    /// Called on the decode queue.
+    func setFrameHandler(_ handler: @escaping @Sendable (DecodedFrame) -> Void)
+    func send(_ input: InputEvent)
+    func close()
+}
+
+public enum StreamEvent: Equatable, Sendable {
+    case hello(version: Int, size: PixelSize)
+    case contentReady
+    case progress(Int)
+    /// Untrusted (from the VM): display only.
+    case pageURL(URL)
+    case ended(EndReason)
+}
+
+public enum EndReason: Equatable, Sendable {
+    /// EOF: the browser crashed, the firewall failed, or the VM's watchdog fired.
+    case closed
+    case protocolViolation(String)
+    case noFirstFrame
+    case decodeErrors
+    case socketError(String)
+}
+
+public enum StreamError: Error, Equatable, Sendable {
+    case versionMismatch(found: Int, need: Int)
+    case protocolViolation(String)
+    case timedOut
+    case refused
+}

Sources/KnotchyCore/Stream/VideoDecoder.swift

@@ -0,0 +1,74 @@
+import CoreMedia
+import CoreVideo
+import Foundation
+import VideoToolbox
+
+final class VideoDecoder {
+    typealias Output = @Sendable (DecodedFrame?, OSStatus) -> Void
+
+    private let session: VTDecompressionSession
+    private let format: CMVideoFormatDescription
+    private let output: Output
+
+    init?(parameterSets: ParameterSets, output: @escaping Output) {
+        var description: CMFormatDescription?
+        let sps = [UInt8](parameterSets.sps)
+        let pps = [UInt8](parameterSets.pps)
+        let status = sps.withUnsafeBufferPointer { s in
+            pps.withUnsafeBufferPointer { p in
+                let pointers = [s.baseAddress!, p.baseAddress!]
+                return CMVideoFormatDescriptionCreateFromH264ParameterSets(
+                    allocator: kCFAllocatorDefault, parameterSetCount: 2, parameterSetPointers: pointers,
+                    parameterSetSizes: [sps.count, pps.count], nalUnitHeaderLength: 4, formatDescriptionOut: &description)
+            }
+        }
+        guard status == noErr, let description else { return nil }
+        let attributes: [CFString: Any] = [
+            kCVPixelBufferPixelFormatTypeKey: kCVPixelFormatType_32BGRA,
+            kCVPixelBufferIOSurfacePropertiesKey: [CFString: Any]() as CFDictionary,
+        ]
+        var created: VTDecompressionSession?
+        let sessionStatus = VTDecompressionSessionCreate(
+            allocator: kCFAllocatorDefault, formatDescription: description, decoderSpecification: nil,
+            imageBufferAttributes: attributes as CFDictionary, outputCallback: nil, decompressionSessionOut: &created)
+        guard sessionStatus == noErr, let created else { return nil }
+        session = created
+        format = description
+        self.output = output
+    }
+
+    deinit {
+        VTDecompressionSessionInvalidate(session)
+    }
+
+    func decode(_ sample: Data) {
+        var block: CMBlockBuffer?
+        guard CMBlockBufferCreateWithMemoryBlock(
+            allocator: kCFAllocatorDefault, memoryBlock: nil, blockLength: sample.count,
+            blockAllocator: kCFAllocatorDefault, customBlockSource: nil, offsetToData: 0,
+            dataLength: sample.count, flags: 0, blockBufferOut: &block) == noErr, let block
+        else { return output(nil, -1) }
+        sample.withUnsafeBytes { bytes in
+            _ = CMBlockBufferReplaceDataBytes(with: bytes.baseAddress!, blockBuffer: block,
+                                              offsetIntoDestination: 0, dataLength: sample.count)
+        }
+        var sampleBuffer: CMSampleBuffer?
+        var size = sample.count
+        guard CMSampleBufferCreateReady(
+            allocator: kCFAllocatorDefault, dataBuffer: block, formatDescription: format,
+            sampleCount: 1, sampleTimingEntryCount: 0, sampleTimingArray: nil,
+            sampleSizeEntryCount: 1, sampleSizeArray: &size, sampleBufferOut: &sampleBuffer) == noErr,
+            let sampleBuffer
+        else { return output(nil, -1) }
+        let output = self.output
+        let status = VTDecompressionSessionDecodeFrame(session, sampleBuffer: sampleBuffer, flags: [],
+                                                       infoFlagsOut: nil) { status, _, image, _, _ in
+            if status == noErr, let image {
+                output(DecodedFrame(image), noErr)
+            } else if status != noErr {
+                output(nil, status)
+            }
+        }
+        if status != noErr { output(nil, status) }
+    }
+}

Sources/KnotchyCore/Stream/WireProtocol.swift

@@ -0,0 +1,214 @@
+import Foundation
+
+/// knotchy-agent ↔ viewer framing, both directions: `[u8 type][u32 big-endian payload length][payload]`.
+public enum WireProtocol {
+    /// Must equal `ARG KNOTCHY_PROTOCOL` in vm/Containerfile. 2: proxy/VPN routes. 3: PAGE_URL.
+    public static let version = 3
+    public static let headerSize = 5
+    public static let maxVideoPayload = 2 * 1024 * 1024
+    /// Also the agent's cap on viewer messages; unknown agent messages up to this size are skipped.
+    public static let maxOtherPayload = 64 * 1024
+    public static let maxPageURL = 16 * 1024
+}
+
+enum MessageType {
+    static let hello: UInt8 = 0x01
+    static let video: UInt8 = 0x02
+    static let progress: UInt8 = 0x03
+    static let navigated: UInt8 = 0x04
+    static let pageURL: UInt8 = 0x05
+    static let pointerMove: UInt8 = 0x10
+    static let pointerButton: UInt8 = 0x11
+    static let scroll: UInt8 = 0x12
+    static let key: UInt8 = 0x13
+    static let navigate: UInt8 = 0x14
+    static let script: UInt8 = 0x15
+    static let keyframe: UInt8 = 0x16
+}
+
+public struct PixelSize: Hashable, Sendable, CustomStringConvertible {
+    public var width: Int
+    public var height: Int
+
+    public init(width: Int, height: Int) {
+        self.width = width
+        self.height = height
+    }
+
+    public var description: String { "\(width)x\(height)" }
+}
+
+public enum AgentMessage: Equatable, Sendable {
+    /// Version 0 means a pre-v1 image, whose HELLO has no version byte.
+    case hello(version: Int, size: PixelSize)
+    case video(size: PixelSize, accessUnit: Data)
+    case progress(Int)
+    case navigated
+    case pageURL(String)
+    case unknown(type: UInt8)
+}
+
+extension AgentMessage {
+    /// For tests and fake agents: out-of-range values trap, so build malformed frames by hand.
+    func encoded() -> Data {
+        var payload = Data()
+        let type: UInt8
+        switch self {
+        case .hello(let version, let size):
+            type = MessageType.hello
+            if version != 0 { payload.append(UInt8(version)) }
+            payload.appendBE16(UInt16(size.width))
+            payload.appendBE16(UInt16(size.height))
+        case .video(let size, let accessUnit):
+            type = MessageType.video
+            payload.appendBE16(UInt16(size.width))
+            payload.appendBE16(UInt16(size.height))
+            payload.append(accessUnit)
+        case .progress(let percent):
+            type = MessageType.progress
+            payload.append(UInt8(percent))
+        case .navigated:
+            type = MessageType.navigated
+        case .pageURL(let url):
+            type = MessageType.pageURL
+            payload.append(Data(url.utf8))
+        case .unknown(let unknownType):
+            type = unknownType
+        }
+        return framed(type, payload)
+    }
+}
+
+func framed(_ type: UInt8, _ payload: Data) -> Data {
+    var message = Data([type])
+    message.appendBE32(UInt32(payload.count))
+    message.append(payload)
+    return message
+}
+
+extension Data {
+    mutating func appendBE16(_ value: UInt16) {
+        append(UInt8(value >> 8))
+        append(UInt8(truncatingIfNeeded: value))
+    }
+
+    mutating func appendBE32(_ value: UInt32) {
+        appendBE16(UInt16(value >> 16))
+        appendBE16(UInt16(truncatingIfNeeded: value))
+    }
+}
+
+/// Strict incremental parser; the VM is untrusted, so any bad message is a final violation.
+/// Call `next()` until nil after each `append`: that keeps the buffer to one message plus one read.
+public struct AgentMessageDecoder: Sendable {
+    public struct ProtocolViolation: Error, Equatable, Sendable, CustomStringConvertible {
+        public let description: String
+
+        init(_ description: String) {
+            self.description = description
+        }
+    }
+
+    public private(set) var helloSize: PixelSize?
+    private var buffer: [UInt8] = []
+    private var start = 0
+    private var violation: ProtocolViolation?
+
+    public init() {}
+
+    public mutating func append(_ bytes: Data) {
+        guard violation == nil else { return }
+        if start > 0 {
+            buffer.removeFirst(start)
+            start = 0
+        }
+        buffer.append(contentsOf: bytes)
+    }
+
+    public var bufferedByteCount: Int { buffer.count - start }
+
+    /// Includes bytes already returned; for tests of the bound.
+    var storedByteCount: Int { buffer.count }
+
+    public mutating func next() throws(ProtocolViolation) -> AgentMessage? {
+        if let violation { throw violation }
+        do {
+            return try decodeNext()
+        } catch {
+            violation = error
+            buffer = []
+            start = 0
+            throw error
+        }
+    }
+
+    private mutating func decodeNext() throws(ProtocolViolation) -> AgentMessage? {
+        guard bufferedByteCount >= WireProtocol.headerSize else { return nil }
+        let type = buffer[start]
+        let length = Int(be32(at: start + 1))
+        try checkHeader(type: type, length: length)
+        guard bufferedByteCount - WireProtocol.headerSize >= length else { return nil }
+
+        let p = start + WireProtocol.headerSize
+        start = p + length
+        switch type {
+        case MessageType.hello:
+            let version = length == 5 ? Int(buffer[p]) : 0
+            let sizeAt = p + length - 4
+            let size = PixelSize(width: Int(be16(at: sizeAt)), height: Int(be16(at: sizeAt + 2)))
+            helloSize = size
+            return .hello(version: version, size: size)
+        case MessageType.video:
+            let size = PixelSize(width: Int(be16(at: p)), height: Int(be16(at: p + 2)))
+            guard size == helloSize else {
+                throw ProtocolViolation("VIDEO is \(size) but HELLO said \(helloSize.map(\.description) ?? "nothing")")
+            }
+            return .video(size: size, accessUnit: Data(buffer[(p + 4)..<(p + length)]))
+        case MessageType.progress:
+            let percent = Int(buffer[p])
+            guard percent <= 100 else { throw ProtocolViolation("PROGRESS \(percent) is over 100") }
+            return .progress(percent)
+        case MessageType.navigated:
+            return .navigated
+        case MessageType.pageURL:
+            guard let url = String(validating: buffer[p..<(p + length)], as: UTF8.self) else {
+                throw ProtocolViolation("PAGE_URL isn't UTF-8")
+            }
+            return .pageURL(url)
+        default:
+            return .unknown(type: type)
+        }
+    }
+
+    private func checkHeader(type: UInt8, length: Int) throws(ProtocolViolation) {
+        func name() -> String { String(format: "type 0x%02x", type) }
+        if helloSize == nil, type != MessageType.hello {
+            throw ProtocolViolation("\(name()) before HELLO")
+        }
+        let valid: Bool
+        switch type {
+        case MessageType.hello:
+            guard helloSize == nil else { throw ProtocolViolation("a second HELLO") }
+            valid = length == 5 || length == 4
+        case MessageType.video:
+            valid = (4...WireProtocol.maxVideoPayload).contains(length)
+        case MessageType.progress:
+            valid = length == 1
+        case MessageType.navigated:
+            valid = length == 0
+        case MessageType.pageURL:
+            valid = (1...WireProtocol.maxPageURL).contains(length)
+        default:
+            valid = length <= WireProtocol.maxOtherPayload
+        }
+        guard valid else { throw ProtocolViolation("\(name()) with a \(length)-byte payload") }
+    }
+
+    private func be16(at i: Int) -> UInt16 {
+        UInt16(buffer[i]) << 8 | UInt16(buffer[i + 1])
+    }
+
+    private func be32(at i: Int) -> UInt32 {
+        UInt32(be16(at: i)) << 16 | UInt32(be16(at: i + 2))
+    }
+}