Dropped Frames Using AudioInputUSB Object

Status
Not open for further replies.

grinch

Well-known member
Hi, I am working on a Teensy 3.6 audio project where I am controlling an array of shift register outputs via a USB audio stream from Max/MSP. I've written a custom object to transfer the data from audio frames to the shift registers via SPI, which is working great, except that I am having an issue with dropped audio frames. I've done a lot of debugging on this front and determined it is not a hardware issue, but that the frame dropping is happening somewhere between the point when audio leaves Max/MSP and the audio data reaches my SPI writing function.

To test this I hooked up a pullup input on the Teensy to be pulled low via a transistor when a certain shift register pin is high. I then write this shift register pin high using my audio stream method (by sending a constant value, corresponding to that pin). If I ever read the pullup pin as high, I know there has been a dropout somewhere in my signal chain. I then added some debugging statements to the process that catches these dropout events to determine the length and frequency of the dropouts.

What I have found by doing this is that the dropout is always the exact length of the sample frame size set in my Max Audio status window, and that the dropouts occur at a fairly regular interval which scales proportionally to frame size (small frame size creates shorter dropouts more often, higher frame size creates longer dropouts less often). Dropout down time can be calculated by the formula:

DropInMs = (1000 / 44117.64706) * FrameSizeInSamples

Dropout rate is roughly 50-70 times the FrameSize, with the ratio growing slightly larger as frame size increases, and always at a regular interval within 1-2 seconds of variation.

I also test the values I am writing to the SPI buffer, which show up as zero during the dropouts, confirming that this is not a hardware issue, but that I'm actually getting an incorrect zero value in software.

These patterns seem to suggest a very specific issue, but I'm not sure where to look next in terms of debugging. Can anyone think of what might be causing this?

Here is the Teensy Sketch I am using to test and debug:
Code:
#include <Audio.h>
#include <Wire.h>
#include <SPI.h>
#include <SD.h>
#include <SerialFlash.h>

AudioInputUSB            usb1;           
AudioOutputAnalog        dac;           //just to trigger audio library update 
AudioOutputCopyBuffer    copyBuffer;
AudioConnection          patchCord1(usb1, 0, dac, 0);
AudioConnection          patchCord2(usb1, 0, copyBuffer, 0);
AudioConnection          patchCord3(usb1, 1, copyBuffer, 1);

SPISettings spiSettings(25000000, MSBFIRST, SPI_MODE0); 
IntervalTimer myTimer;

#define SS_PIN 10
#define PULLUP_PIN 33

int16_t out[2];

void doOutput(){
  __disable_irq();
    digitalWriteFast(SS_PIN, HIGH);
    copyBuffer.readFromBuffers(&out[0], &out[1]);
    digitalWriteFast(SS_PIN, LOW);
    SPI.transfer16(out[1]); 
    SPI.transfer16(out[0]);  
  __enable_irq();
}

void setup() {                
  AudioMemory(100);
  Serial.begin(115200);
  pinMode(PULLUP_PIN, INPUT_PULLUP);
  pinMode(SS_PIN, OUTPUT);
  digitalWriteFast(SS_PIN, LOW);
  SPI.begin();
  SPI.beginTransaction(spiSettings);
  myTimer.priority(0);
  myTimer.begin(doOutput, 22.666666666062222);
  copyBuffer.begin();
}

unsigned long blip = 0;
unsigned long lastBlip = 0;
unsigned long blipInterval = 0;
int16_t checkOut[2] = {};

void loop() {
  if(copyBuffer.checkOverrunAndClear()){
    Serial.println("Error! Overrun!");
    Serial.println("");
    delay(100);
  }
  if(digitalRead(PULLUP_PIN)){
    blip = micros();
    blipInterval = micros() - lastBlip;
    lastBlip = micros();
    checkOut[0] = out[0];
    checkOut[1] = out[1];
    while(digitalRead(PULLUP_PIN)){ /*wait*/ }
    blip = micros() - blip;
    Serial.println("Error! Blip Detected!");
    Serial.print("Down Time: ");
    Serial.print(float(blip) / 1000.0f, 6);
    Serial.println(" ms");
    Serial.print("Blip Interval: ");
    Serial.print(float(blipInterval) / 1000.0f, 6);
    Serial.println(" ms");
    Serial.print("Out Reg 0 Val: ");
    Serial.println(checkOut[0]);
    Serial.print("Out Reg 1 Val: ");
    Serial.println(checkOut[1]);
    Serial.println("");
  }
}

Here the class I am using to transfer audio buffers to the SPI output:
output_copybuffer.h:
Code:
#ifndef output_copybuffer_h_
#define output_copybuffer_h_

#define COPY_BUFFER_COUNT 16

#include "Arduino.h"
#include "AudioStream.h"
#include "DMAChannel.h"

class AudioOutputCopyBuffer : public AudioStream
{
public:
	AudioOutputCopyBuffer(void) : AudioStream(2, inputQueueArray) { begin(); }
	virtual void update(void);
	void begin(void);
	void readFromBuffers(int16_t *p1, int16_t *p2);
	void readFromBuffersAlternating(int16_t *p1, int16_t *p2, int16_t *p3, int16_t *p4);
	bool checkOverrunAndClear(){
		bool check = error;
		error = false;
		return check;
	};
private:
	int16_t leftBuffer[COPY_BUFFER_COUNT][AUDIO_BLOCK_SAMPLES];
	int16_t rightBuffer[COPY_BUFFER_COUNT][AUDIO_BLOCK_SAMPLES];
	int16_t write, read;
	int16_t index = 0;
	bool error = false;
	audio_block_t *inputQueueArray[2];
	static bool update_responsibility;
};
#endif

output_copybuffer.cpp
Code:
#include <Arduino.h>
#include "output_copybuffer.h"
#include "utility/pdb.h"

bool AudioOutputCopyBuffer::update_responsibility = false;

void AudioOutputCopyBuffer::begin(void)
{
	write = 0;
	read = 0;
	error = false;
	index = 0;
}


void AudioOutputCopyBuffer::update(void)
{
	audio_block_t *b1;
	audio_block_t *b2;
	b1 = receiveReadOnly(0); // input 0
	b2 = receiveReadOnly(1); // input 1
	if (!b1 || !b2) {
		return;
	}

	__disable_irq();
	memcpy(leftBuffer[write], b1->data, AUDIO_BLOCK_SAMPLES * 2);
	memcpy(rightBuffer[write], b2->data, AUDIO_BLOCK_SAMPLES * 2);
	write++;
	if(write >= COPY_BUFFER_COUNT){ write = 0;}
	if(write == read){ error = true; }
	__enable_irq();
	release(b1);
	release(b2);
}


void AudioOutputCopyBuffer::readFromBuffers(int16_t *p1, int16_t *p2)
{
	if(write != read){
		*p1 = leftBuffer[read][index];
		*p2 = rightBuffer[read][index];
		index++;
		if(index >= AUDIO_BLOCK_SAMPLES){
			index = 0;
			read++;
			if(read >= COPY_BUFFER_COUNT){
				read = 0;
			}
		}
	}else{
		error = true;
	}
}

Here are the results of my debug print statements at different buffer sizes:
Code:
128 Frame Size:

Error! Blip Detected!
Down Time: 2.900000 ms
Blip Interval: 7189.503906 ms
Out Reg 0 Val: 0
Out Reg 1 Val: 0

Error! Blip Detected!
Down Time: 2.901000 ms
Blip Interval: 8640.170898 ms
Out Reg 0 Val: 0
Out Reg 1 Val: 0

Error! Blip Detected!
Down Time: 2.900000 ms
Blip Interval: 7148.884766 ms
Out Reg 0 Val: 0
Out Reg 1 Val: 0

Error! Blip Detected!
Down Time: 2.901000 ms
Blip Interval: 7642.111816 ms
Out Reg 0 Val: 0
Out Reg 1 Val: 0

Error! Blip Detected!
Down Time: 2.900000 ms
Blip Interval: 7917.738770 ms
Out Reg 0 Val: 0
Out Reg 1 Val: 0


256 Frame Size:

Error! Blip Detected!
Down Time: 5.802000 ms
Blip Interval: 15231.999023 ms
Out Reg 0 Val: 0
Out Reg 1 Val: 0

Error! Blip Detected!
Down Time: 5.802000 ms
Blip Interval: 15707.820313 ms
Out Reg 0 Val: 0
Out Reg 1 Val: 0

Error! Blip Detected!
Down Time: 5.802000 ms
Blip Interval: 15319.039063 ms
Out Reg 0 Val: 0
Out Reg 1 Val: 0

Error! Blip Detected!
Down Time: 5.803000 ms
Blip Interval: 15673.002930 ms
Out Reg 0 Val: 0
Out Reg 1 Val: 0


512 Frame Size:

Error! Blip Detected!
Down Time: 11.604000 ms
Blip Interval: 30788.949219 ms
Out Reg 0 Val: 0
Out Reg 1 Val: 0

Error! Blip Detected!
Down Time: 11.604000 ms
Blip Interval: 30649.683594 ms
Out Reg 0 Val: 0
Out Reg 1 Val: 0

Error! Blip Detected!
Down Time: 11.604000 ms
Blip Interval: 30928.212891 ms
Out Reg 0 Val: 0
Out Reg 1 Val: 0


1024 Frame Size:

Error! Blip Detected!
Down Time: 23.209000 ms
Blip Interval: 61856.429687 ms
Out Reg 0 Val: 0
Out Reg 1 Val: 0

Error! Blip Detected!
Down Time: 23.211000 ms
Blip Interval: 62158.164062 ms
Out Reg 0 Val: 0
Out Reg 1 Val: 0

Error! Blip Detected!
Down Time: 23.209999 ms
Blip Interval: 61902.847656 ms
Out Reg 0 Val: 0
Out Reg 1 Val: 0

Error! Blip Detected!
Down Time: 23.209000 ms
Blip Interval: 60626.261719 ms
Out Reg 0 Val: 0
Out Reg 1 Val: 0


2048 Frame Size:

Error! Blip Detected!
Down Time: 46.419998 ms
Blip Interval: 123109.375000 ms
Out Reg 0 Val: 0
Out Reg 1 Val: 0

Error! Blip Detected!
Down Time: 46.419998 ms
Blip Interval: 124734.117188 ms
Out Reg 0 Val: 0
Out Reg 1 Val: 0

Error! Blip Detected!
Down Time: 46.419998 ms
Blip Interval: 124177.062500 ms
Out Reg 0 Val: 0
Out Reg 1 Val: 0

Error! Blip Detected!
Down Time: 46.419998 ms
Blip Interval: 123202.218750 ms
Out Reg 0 Val: 0
Out Reg 1 Val: 0

Error! Blip Detected!
Down Time: 46.421001 ms
Blip Interval: 124594.859375 ms
Out Reg 0 Val: 0
Out Reg 1 Val: 0
 
Here is the Max Patch I am using to test this. It sends a constant value of 8.0 / 32767.0 out of the left audio channel, which gets read as 0B00001000 in binary on the Teensy end, which corresponds to the 4th pin of my shift register being written high (this is the pin I am using to test dropouts). The right input receives an oscillating signal. I am also using an edge detection object to ensure that the constant value going to the left channel is not changing within the patch. At some point between this constant signal leaving the patch and the audio buffer reaching my SPI write function a frame gets dropped and a period of zero values gets inserted.

Patcher Screenshot
 
@grinch

I noticed something similar when I tried to add USB audio out of my TeensyMIDIPolySynth (https://forum.pjrc.com/threads/60690-queued-TeensyMIDIPolySynth?p=237404&viewfull=1#post237404). In my case, I would get pauses in the output stream & then very quick "catch-up audio". I did find that the number of occurrences of the pauses in my project varied with the optimization level (Faster, Fast, Fastest, Debug, Smallest Code) chosen at compile time (I was using "Fastest" when I noticed this, but it turns out that using "Debug" optimization almost completely eliminates the pauses for me). I don't have any suggestions on the possible cause(s), but you might try compiling with the different optimization levels & see what effect that might have on your dropouts, if any.

Mark J Culross
KD5RXT
 
I don't have any suggestions on the possible cause(s), but you might try compiling with the different optimization levels & see what effect that might have on your dropouts, if any.

Hmmm... tired that out, doesn't seem to have much effect. When I compiled as debug there was a lag before dropouts started occurring but then they come back at the same rate as before. Was worth a shot though.
 
I'm really hoping Paul or someone else with in depth knowledge of the audio library can weigh in on this. My code is doing a multithreaded thing with two layers of interrupts, and I'm thinking that the dropouts may be a result of that. Having someone with in depth knowledge weigh in would be really helpful in terms of figuring out where to look for that issue.
 
* Your IntervalTimer interrupt may have a higher priority than the SWI that triggers all the audio class update() functions. That might cause some funny interactions if the former interrupts the latter.

* Your out[] array is accessed in both ISR and non-ISR code, yet is not declared 'volatile'.

* I'd protect the non-ISR accesses to out[] by temporarily disabling audio interrupts:
Code:
   AudioInterrupts();
   checkOut[0] = out[0];
   checkOut[1] = out[1];
   AudioNoInterrupts();

* If it were me, I'd nail down the interrupt for the doOutput() function to be exactly synchronous with the audio frame update rate by manually programming a PIT. Paul said in your previous thread that you didn't need to do this. So, I have no doubt that he's right. But, it's a theoretical loose end that bothers me philosophically. So, I'd do it for my OCD. Besides, it can't hurt.
 
* Your IntervalTimer interrupt may have a higher priority than the SWI that triggers all the audio class update() functions. That might cause some funny interactions if the former interrupts the latter.

This is the case. I set my interval timer priority to 0 in order to ensure it is completely sample accurate. I'm not exactly sure how this interacts with other timer priority stuff, but it is the highest priority that object allows. My assumption would be that this makes this interrupt the highest priority out of all interrupts on the system, but perhaps someone can confirm this understanding.

I do make sure in my buffer transfer object to disable interrupts while copying the buffers over, but it may be that there is another interaction elsewhere in the audio library that is messing things up.

* Your out[] array is accessed in both ISR and non-ISR code, yet is not declared 'volatile'.

* I'd protect the non-ISR accesses to out[] by temporarily disabling audio interrupts:

Good catch. I will fix those and see if it changes anything. The second thing only happens in the case that a dropout is already detected, so I doubt its affecting anything, but I'll change it anyway because it seems like good practice.

* If it were me, I'd nail down the interrupt for the doOutput() function to be exactly synchronous with the audio frame update rate by manually programming a PIT. Paul said in your previous thread that you didn't need to do this. So, I have no doubt that he's right. But, it's a theoretical loose end that bothers me philosophically. So, I'd do it for my OCD. Besides, it can't hurt.

This is something I'm pretty interested in, I think it may provide a solution. There are timers that are used by the audio library to initiate DMA transfers, and it seems like these are exactly timed to be sample accurate. I know a PDB timer is used for the DAC output from looking at the source code. There is also probably some timer that the exact Teensy 3.6 sample rate of 44117.64706 is derived from.

Can anyone suggest how I should get started in setting up an interrupt using one of these sample accurate timing sources?
 
The simplest solution may be to move to the T4.1 (or any T4), which has accurate 44.1kHz sampling rates.

I got caught with dropped frames in the Ethernet library when transferring samples between T3's and T4's due to the slightly different sample rates.

One way to check this, is to change the CPU frequency on your T3.6 (180 & 168 MHz have different sample rates) to see if there's any impact.

T3.6:

Code:
[U]Clock  Sample freq   Deviation[/U]
[B]180    44116.99[/B]      -0.0385%
[B]168    44128.77[/B]      -0.0652%
144    44116.99      -0.0385%
120    44116.99      -0.0385%

Hope this helps.
 
The simplest solution may be to move to the T4.1 (or any T4), which has accurate 44.1kHz sampling rates

Yeah I was already thinking of trying this. Got a couple Teensy 4.0s in the mail, and I'm going to see if it works any better once I switch them out.

I've been looking on github for the Teensy4 core equivalent of this file just to get a jump on researching things, but I can seem to find it, does anyone know where it's located? AudioStream.h
 
Before going too far down the road of trying new hardware, you should take the divide and conquer approach to debug the problem. I'd replace the AudioInputUSB object with a couple of AudioPlayQueue objects that you populate (quickly) in your loop() code with the special test pattern. This should tell you whether the problem is the timing of your custom Audio class or interaction with the AudioInputUSB class.
 
Before going too far down the road of trying new hardware, you should take the divide and conquer approach to debug the problem. I'd replace the AudioInputUSB object with a couple of AudioPlayQueue objects that you populate (quickly) in your loop() code with the special test pattern. This should tell you whether the problem is the timing of your custom Audio class or interaction with the AudioInputUSB class.

I tried this using the AudioSynthWaveformDC object to generate a constant value for dropout detection output. As I would expect, this version detects no dropouts whatsoever. Left it running over an hour to test this. This is definitely related to AudioInputUSB timing.

Test Code:
Code:
#include <Audio.h>
#include <Wire.h>
#include <SPI.h>
#include <SD.h>
#include <SerialFlash.h>

AudioSynthWaveformDc     dc;     
AudioSynthWaveformDc     dc2;      
AudioOutputAnalog        dac;           //just to trigger audio library update 
AudioOutputCopyBuffer    copyBuffer;
AudioConnection          patchCord1(dc, 0, dac, 0);
AudioConnection          patchCord2(dc, 0, copyBuffer, 0);
AudioConnection          patchCord3(dc2, 0, copyBuffer, 1);

SPISettings spiSettings(25000000, MSBFIRST, SPI_MODE0); 
IntervalTimer myTimer;

#define SS_PIN 10
#define PULLUP_PIN 33

volatile int16_t out[2];

void doOutput(){
  __disable_irq();
    digitalWriteFast(SS_PIN, HIGH);
    copyBuffer.readFromBuffers(&out[0], &out[1]);
    digitalWriteFast(SS_PIN, LOW);
    SPI.transfer16(out[1]); 
    SPI.transfer16(out[0]);  
  __enable_irq();
}

void setup() {                
  AudioMemory(100);
  Serial.begin(115200);
  pinMode(PULLUP_PIN, INPUT_PULLUP);
  pinMode(SS_PIN, OUTPUT);
  digitalWriteFast(SS_PIN, LOW);
  dc.amplitude(8.0f / 32767.0f);
  dc2.amplitude(1.0f / 32767.0f);
  SPI.begin();
  SPI.beginTransaction(spiSettings);
  myTimer.priority(0);
  myTimer.begin(doOutput, 22.666666666062222); //22.666666666062222
  copyBuffer.begin();
}

unsigned long blip = 0;
unsigned long lastBlip = 0;
unsigned long blipInterval = 0;
int16_t checkOut[2] = {};

void loop() {
  if(copyBuffer.checkOverrunAndClear()){
    Serial.println("Error! Overrun!");
    Serial.println("");
    delay(100);
  }
  if(digitalRead(PULLUP_PIN)){
    blip = micros();
    blipInterval = micros() - lastBlip;
    lastBlip = micros();
    AudioNoInterrupts();
    checkOut[0] = out[0];
    checkOut[1] = out[1];
    AudioInterrupts();
    while(digitalRead(PULLUP_PIN)){ /*wait*/ }
    blip = micros() - blip;
    Serial.println("Error! Blip Detected!");
    Serial.print("Down Time: ");
    Serial.print(float(blip) / 1000.0f, 6);
    Serial.println(" ms");
    Serial.print("Blip Interval: ");
    Serial.print(float(blipInterval) / 1000.0f, 6);
    Serial.println(" ms");
    Serial.print("Out Reg 0 Val: ");
    Serial.println(checkOut[0]);
    Serial.print("Out Reg 1 Val: ");
    Serial.println(checkOut[1]);
    Serial.println("");
  }
}
 
The simplest solution may be to move to the T4.1 (or any T4), which has accurate 44.1kHz sampling rates.

So I tried this with the Teensy 4.0, which seems to solve whatever dropout issue I was having with the Teensy 3.6. I am getting the occasional buffer overrun flag, which seems to be related to the limited resolution of the PIT used by the IntervalTimer object to generate interrupts. This doesn't have much of an effect on output quality however as its just an occasional repeated sample rather than a full on dropout, so I'd call it a win. Working example code here:

Code:
#include <Audio.h>
#include <Wire.h>
#include <SPI.h>
#include <SD.h>
#include <SerialFlash.h>
#include "output_copybuffer.h"

AudioInputUSB            usb1;           
AudioOutputMQS           mqs;           //just to trigger audio library update 
AudioOutputCopyBuffer    copyBuffer;
AudioConnection          patchCord1(usb1, 0, mqs, 0);
AudioConnection          patchCord2(usb1, 0, copyBuffer, 0);
AudioConnection          patchCord3(usb1, 1, copyBuffer, 1);

SPISettings spiSettings(25000000, MSBFIRST, SPI_MODE0); 
IntervalTimer myTimer;

#define SS_PIN 10
#define PULLUP_PIN 14

volatile int16_t out[2];

bool alt = false;

void doOutput(){
  __disable_irq();
  if(alt){
    digitalWriteFast(SS_PIN, HIGH);
  }else{
    copyBuffer.readFromBuffers(&out[0], &out[1]);
    digitalWriteFast(SS_PIN, LOW);
    SPI.transfer16(out[1]); 
    SPI.transfer16(out[0]);  
  }
  alt = !alt;
  __enable_irq();
}

void setup() {                
  AudioMemory(100);
  Serial.begin(115200);
  pinMode(PULLUP_PIN, INPUT_PULLUP);
  pinMode(SS_PIN, OUTPUT);
  digitalWriteFast(SS_PIN, LOW);
  SPI.begin();
  SPI.beginTransaction(spiSettings);
  myTimer.priority(0);
  myTimer.begin(doOutput, 11.4); //11.337868480725624 //22.675736961451247 //22.666666666062222
  copyBuffer.begin();
}

unsigned long blip = 0;
unsigned long lastBlip = 0;
unsigned long blipInterval = 0;
int16_t checkOut[2] = {};
unsigned long counter = 0;

unsigned long getOverrun;

void loop() {

  if(millis() - counter > 1000){
    if(copyBuffer.checkOverrunAndClear()){
      Serial.println("Error! Overrun!");
      Serial.println("");
    }
    counter = millis();
  }

  if(digitalRead(PULLUP_PIN)){
    blip = micros();
    blipInterval = micros() - lastBlip;
    lastBlip = micros();
    AudioNoInterrupts();
    checkOut[0] = out[0];
    checkOut[1] = out[1];
    AudioInterrupts();
    while(digitalRead(PULLUP_PIN)){ /*wait*/ }
    blip = micros() - blip;
    Serial.println("Error! Blip Detected!");
    Serial.print("Down Time: ");
    Serial.print(float(blip) / 1000.0f, 6);
    Serial.println(" ms");
    Serial.print("Blip Interval: ");
    Serial.print(float(blipInterval) / 1000.0f, 6);
    Serial.println(" ms");
    Serial.print("Out Reg 0 Val: ");
    Serial.println(checkOut[0]);
    Serial.print("Out Reg 1 Val: ");
    Serial.println(checkOut[1]);
    Serial.println("");
  }
}

However, switching to the Teensy 4.0 has created a problem with a 4 channel encoding technique I was using successfully on the Teensy 3.6 (other than the error we're discussing here), due to the different handling of usb audio frame sync on the 4.0. Made separate post about it, if anyone is able to help with that it would be much appreciated. I am very close on this project and would be really excited to see it working. Post about it here:

POST
 
Status
Not open for further replies.
Back
Top