occasional glitches and empty buffers when using AudioFileStream + AVAudioConverter

I'm streaming mp3 audio data using URLSession/AudioFileStream/AVAudioConverter and getting occasional silent buffers and glitches (little bleeps and whoops as opposed to clicks). The issues are present in an offline test, so this isn't an issue of underruns.

Doing some buffering on the input coming from the URLSession (URLSessionDataTask) reduces the glitches/silent buffers to rather infrequent, but they do still happen occasionally.

    var bufferedData = Data()

    func parseBytes(data: Data) {

        bufferedData.append(data)

        // XXX: this buffering reduces glitching
        //      to rather infrequent. But why?
        if bufferedData.count > 32768 {

            bufferedData.withUnsafeBytes { (bytes: UnsafeRawBufferPointer) in
                guard let baseAddress = bytes.baseAddress else { return }

                let result = AudioFileStreamParseBytes(audioStream!,
                                                       UInt32(bufferedData.count),
                                                       baseAddress,
                                                       [])

                if result != noErr {
                    print("❌ error parsing stream: \(result)")
                }

            }

            bufferedData = Data()

        }
    }

No errors are returned by AudioFileStream or AVAudioConverter.

    func handlePackets(data: Data,
                       packetDescriptions: [AudioStreamPacketDescription]) {

        guard let audioConverter else {
            return
        }

        var maxPacketSize: UInt32 = 0
        for packetDescription in packetDescriptions {
            maxPacketSize = max(maxPacketSize, packetDescription.mDataByteSize)

            if packetDescription.mDataByteSize == 0 {
                print("EMPTY PACKET")
            }

            if Int(packetDescription.mStartOffset) + Int(packetDescription.mDataByteSize) > data.count {
                print("❌ Invalid packet: offset \(packetDescription.mStartOffset) + size \(packetDescription.mDataByteSize) > data.count \(data.count)")
            }
        }

        let bufferIn = AVAudioCompressedBuffer(format: inFormat!, packetCapacity: AVAudioPacketCount(packetDescriptions.count), maximumPacketSize: Int(maxPacketSize))
        bufferIn.byteLength = UInt32(data.count)

        for i in 0 ..< Int(packetDescriptions.count) {
            bufferIn.packetDescriptions![i] = packetDescriptions[i]
        }
        bufferIn.packetCount = AVAudioPacketCount(packetDescriptions.count)

        _ = data.withUnsafeBytes { ptr in
            memcpy(bufferIn.data, ptr.baseAddress, data.count)
        }

        if verbose {
            print("handlePackets: \(data.count) bytes")
        }

        // Setup input provider closure
        var inputProvided = false
        let inputBlock: AVAudioConverterInputBlock = { packetCount, statusPtr in
            if !inputProvided {
                inputProvided = true
                statusPtr.pointee = .haveData
                return bufferIn
            } else {
                statusPtr.pointee = .noDataNow
                return nil
            }
        }

        // Loop until converter runs dry or is done
        while true {
            let bufferOut = AVAudioPCMBuffer(pcmFormat: outFormat, frameCapacity: 4096)!
            bufferOut.frameLength = 0

            var error: NSError?
            let status = audioConverter.convert(to: bufferOut, error: &error, withInputFrom: inputBlock)

            switch status {
            case .haveData:
                if verbose {
                    print("✅ convert returned haveData: \(bufferOut.frameLength) frames")
                }
                if bufferOut.frameLength > 0 {

                    if bufferOut.isSilent {
                        print("(haveData) SILENT BUFFER at frame \(totalFrames), pending: \(pendingFrames), inputPackets=\(bufferIn.packetCount), outputFrames=\(bufferOut.frameLength)")
                    }

                    outBuffers.append(bufferOut)
                    totalFrames += Int(bufferOut.frameLength)
                }

            case .inputRanDry:

                if verbose {
                    print("🔁 convert returned inputRanDry: \(bufferOut.frameLength) frames")
                }

                if bufferOut.frameLength > 0 {

                    if bufferOut.isSilent {
                        print("(inputRanDry) SILENT BUFFER at frame \(totalFrames), pending: \(pendingFrames), inputPackets=\(bufferIn.packetCount), outputFrames=\(bufferOut.frameLength)")
                    }

                    outBuffers.append(bufferOut)
                    totalFrames += Int(bufferOut.frameLength)
                }
                
                return // wait for next handlePackets

            case .endOfStream:

                if verbose {
                    print("✅ convert returned endOfStream")
                }
                return

            case .error:

                if verbose {
                    print("❌ convert returned error")
                }

                if let error = error {
                    print("error converting: \(error.localizedDescription)")
                }
                return

            @unknown default:
                fatalError()
            }
        }

    }