Two queues on i2s input not working

TeensyPhonon · Apr 24, 2024

I have a code for Teensy 4.1 that is working well with one SPH0645 microphone (mono). But now, I need to make it stereo, which is why I am here...

The strangest part is that even the output queue stops working (seems to struggle at "sp_L = Q_out_L.getBuffer();").

C:

#include <Audio.h>
#include <arm_const_structs.h>
#include <utility/imxrt_hw.h>
#include <SD.h>
#include <SPI.h>
#include "input_i2s_nobegin.h"
#include "output_i2s_nobegin.h"


File                         dataFile;
const int                    chipSelect = BUILTIN_SDCARD;          

extern "C" uint32_t set_arm_clock(uint32_t frequency);
const float32_t PROGMEM audio_gain = 0.05;

const int Npartitions = 500;
const double PROGMEM SAMPLE_RATE = AUDIO_SAMPLE_RATE_EXACT;
const int PROGMEM partitionsize = AUDIO_BLOCK_SAMPLES;

const float32_t T = Npartitions*partitionsize/SAMPLE_RATE;
const int nc = T*SAMPLE_RATE;
const int N_memory = 10;
float32_t mem_delay = N_memory*partitionsize/SAMPLE_RATE;

int16_t *sp_L;
uint8_t first_block = 1;
const int PROGMEM nfor = nc / partitionsize;

float32_t DMAMEM float_buffer_L[nc];

char buffer[4];
float32_t chirp[nc];

AudioInputI2SNoBegin        i2s_in;
AudioRecordQueue            Q_in_L;
AudioRecordQueue            Q_in_R;
AudioPlayQueue              Q_out_L;
AudioOutputI2SNoBegin       i2s_out;
AudioConnection             patchCord_in(i2s_in, 0, Q_in_L, 0);
AudioConnection             patchCord_in_R(i2s_in, 1, Q_in_R, 0);
AudioConnection             patchCord_out(Q_out_L, 0,  i2s_out, 0);

String command;


void setup() {

  SD.begin(chipSelect);
 

  AudioMemory(N_memory);
  delay(100);

  Q_in_L.begin();
  Q_in_R.begin();
  pinMode(0,INPUT_PULLDOWN);
  pinMode(1,OUTPUT);
  pinMode(2,OUTPUT);
  pinMode(LED_BUILTIN,OUTPUT);

  setI2SFreq(SAMPLE_RATE);

  digitalWrite(1,HIGH);
  digitalWrite(LED_BUILTIN,HIGH);

  Serial.begin(115200);
  while(!Serial);

  Serial.println(SAMPLE_RATE);
  Serial.println(partitionsize);
  Serial.println(Npartitions);

}

void loop() {
  //stuff
}

AudioInputI2SNoBegin and AudioOutputI2SNoBegin are two custom objects that are identical to the original I2S objects but do not call "begin()" (which I call later) when created. The problem is the same with the original I2S objects.

If you comment out the line "AudioConnection patchCord_in_R(i2s_in, 1, Q_in_R, 0);" (or Q_in_R.begin()), it works! Note that the two channels are working separately (by changing "patchCord_in(i2s_in, 0, Q_in_L, 0);" to "patchCord_in(i2s_in, 1, Q_in_L, 0);"). I also made a test code... which works:

C:

#include <Audio.h>
#include <Wire.h>
#include <SPI.h>
#include <SD.h>
#include <SerialFlash.h>

// GUItool: begin automatically generated code
AudioInputI2S            i2s1;           //xy=204,228
AudioRecordQueue         queue2;         //xy=438,297
AudioRecordQueue         queue1;         //xy=440,162
AudioConnection          patchCord1(i2s1, 0, queue1, 0);
AudioConnection          patchCord2(i2s1, 1, queue2, 0);
// GUItool: end automatically generated code

void setup() {
  AudioMemory(10);
  delay(100);
  queue1.begin();
  queue2.begin();
}

void loop() {
 
  if(queue1.available()){
    Serial.println("left channel OK");
    queue1.freeBuffer();
  }
 
  if(queue2.available()){
    Serial.println("right channel OK");
    queue2.freeBuffer();
  }
 
}

I really don't see where the problem is...

h4yn0nnym0u5e · Apr 24, 2024

The problem is almost certainly in //stuff, where we can't see it. In normal code I'd say you've allocated far too few audio blocks, and started the queues way too early, so they consume all the blocks before you even get to loop(). It may be different with your edited I²S objects, but probably not.

Your test code probably isn't working either - you never call AudioRecordQueue::readBuffer(), so the calls to freeBuffer() will be doing nothing (you can't free what you haven't read), and of course your queues always have data available, you never read them...

TeensyPhonon · Apr 24, 2024

I corrected the test code, which still works.

C:

#include <Audio.h>
#include <Wire.h>
#include <SPI.h>
#include <SD.h>
#include <SerialFlash.h>

int16_t *sp;
float float_buffer[128];

// GUItool: begin automatically generated code
AudioInputI2S            i2s1;           //xy=204,228
AudioRecordQueue         queue2;         //xy=438,297
AudioRecordQueue         queue1;         //xy=440,162
AudioConnection          patchCord1(i2s1, 0, queue1, 0);
AudioConnection          patchCord2(i2s1, 1, queue2, 0);
// GUItool: end automatically generated code

void setup() {
  AudioMemory(10);
  delay(100);
  queue1.begin();
  queue2.begin();
}

void loop() {
  
  if((queue1.available()) && (queue2.available())){
    sp = queue1.readBuffer();
    arm_q15_to_float (sp, float_buffer, 128);
    queue1.freeBuffer();
    Serial.print(float_buffer[0]);
    Serial.print(", ");
  
    sp = queue2.readBuffer();
    arm_q15_to_float (sp, float_buffer, 128);
    queue2.freeBuffer();
    Serial.println(float_buffer[0]);
  }
 
}

Here is the part of my code in loop() where the problem happens:

C:

void loop() {

  while(!Serial);
  while(!Serial.available());
  command = Serial.readString();

  if(command.startsWith("begin_i2s")){
    delay(100);
    digitalWrite(1,LOW);
    i2s_in.begin();
    i2s_out.begin();
    delay(100);
    digitalWrite(1,HIGH);
    Q_in_L.begin();
    Q_in_R.begin();
  }

  if(command.startsWith("play")){

    digitalWrite(1,LOW);
    
    for (int i = 0; i < nfor; i++) {
      sp_L = Q_out_L.getBuffer();
      arm_float_to_q15 (&chirp[partitionsize * i], sp_L, partitionsize);
      Q_out_L.playBuffer();
    }

    digitalWrite(1,HIGH);
  }
}

The commands are sent (they can wait for a long time because they are sent on demand) via a Python code on a PC.

I moved Q_in_L.begin(); and Q_in_R.begin(); in loop(), but it's still the same...

If it is due to the queues filling up, I don't understand why it was working even with only one...

h4yn0nnym0u5e · Apr 24, 2024

Your test code didn't work before, but I can see no reason why it wouldn't now.

Your "part of " loop() clearly has problems. Let's assume Serial is actually connected, so while (!Serial); doesn't cause an issue. Let's assume you've sent the begin_i2s command so I²S input and output are generating blocks. Now while(!Serial.available()); will cause problems, it'll block queue emptying, so your 10 blocks will fill up, and no more audio will be put into them.

Say you then send play: you'll fill Q_out_L - except, probably you won't, because getBuffer() will fail and return a null pointer, because no blocks are available. arm_float_to_q15() will attempt to fill the non-existent buffer, and your Teensy will crash.

What about Q_in_L and Q_in_R? No idea - if that's your whole loop, you begin() them and never touch them thereafter, so no matter how many blocks to allocate they'll rapidly be consumed.

It is totally impossible to help you unless you submit a complete, compilable sketch exhibiting the issue. "Part of" is just not enough...

TeensyPhonon · Apr 25, 2024

Here is the mnimal code (using the original i2s objects) reproducing the problem:

C:

#include <Audio.h>
#include <arm_const_structs.h>
#include <utility/imxrt_hw.h>
#include <SD.h>
#include <SPI.h>


File                         dataFile;
const int                    chipSelect = BUILTIN_SDCARD;     

extern "C" uint32_t set_arm_clock(uint32_t frequency);
const float32_t PROGMEM audio_gain = 0.05; 

const int Npartitions = 500;
const double PROGMEM SAMPLE_RATE = AUDIO_SAMPLE_RATE_EXACT; 
const int PROGMEM partitionsize = AUDIO_BLOCK_SAMPLES; 

const float32_t T = Npartitions*partitionsize/SAMPLE_RATE; 
const int nc = T*SAMPLE_RATE;
const int N_memory = 10;
float32_t mem_delay = N_memory*partitionsize/SAMPLE_RATE;

int16_t *sp_L;
uint8_t first_block = 1;
const int PROGMEM nfor = nc / partitionsize;

float32_t DMAMEM float_buffer_L[nc];

char buffer[4];
float32_t chirp[nc];

AudioInputI2S               i2s_in; 
AudioRecordQueue            Q_in_L; 
AudioRecordQueue            Q_in_R;
AudioPlayQueue              Q_out_L;
AudioOutputI2S              i2s_out; 
AudioConnection             patchCord_in(i2s_in, 0, Q_in_L, 0);
AudioConnection             patchCord_in_R(i2s_in, 1, Q_in_R, 0);
AudioConnection             patchCord_out(Q_out_L, 0,  i2s_out, 0);

String command;


void setup() {

  SD.begin(chipSelect); 
 

  AudioMemory(N_memory);
  delay(100);

 
  pinMode(0,INPUT_PULLDOWN);
  pinMode(1,OUTPUT);
  pinMode(2,OUTPUT);
  pinMode(LED_BUILTIN,OUTPUT); 

  setI2SFreq(SAMPLE_RATE);

  digitalWrite(1,HIGH); 
  digitalWrite(LED_BUILTIN,HIGH);

  Serial.begin(115200);
  while(!Serial);

  Serial.println(SAMPLE_RATE);
  Serial.println(partitionsize);
  Serial.println(Npartitions);

  Q_in_L.begin();
  Q_in_R.begin();

  //dummy signal for test
  for(int i = 0;i<nc;i++){
    chirp[i] = 0.1*sin(TWO_PI*440*(float)i/SAMPLE_RATE);
  }
 
}

            

void loop() {

  if(Serial.available()){
    command = Serial.readString();

    
    if(command.startsWith("play")){

      digitalWrite(1,LOW);
      
      for (int i = 0; i < nfor; i++) {
        sp_L = Q_out_L.getBuffer();
        arm_float_to_q15 (&chirp[partitionsize * i], sp_L, partitionsize);
        Q_out_L.playBuffer();
      }

      digitalWrite(1,HIGH);
    }
      
  }

}


void setI2SFreq(int freq) {
  int n1 = 4;
  int n2 = 1 + (24000000 * 27) / (freq * 256 * n1);
  double C = ((double)freq * 256 * n1 * n2) / 24000000;
  int c0 = C;
  int c2 = 10000;
  int c1 = C * c2 - (c0 * c2);
  set_audioClock(c0, c1, c2, true);
  CCM_CS1CDR = (CCM_CS1CDR & ~(CCM_CS1CDR_SAI1_CLK_PRED_MASK | CCM_CS1CDR_SAI1_CLK_PODF_MASK)) | CCM_CS1CDR_SAI1_CLK_PRED(n1-1) | CCM_CS1CDR_SAI1_CLK_PODF(n2-1);
}

If I send "play" with the serial monitor, nothing happens. If I comment Q_in_R.begin();, it works as expected.

I tried replacing while() and clearing queues in loop(), but it had no effect...

PaulStoffregen · Apr 25, 2024

Try using AudioMemoryUsage() to monitor how many of the 10 audio memory blocks are used up. Or use AudioMemoryUsageMax() to check whether you have at any point used up all the memory.

My guess, admittedly from only a quick glace at your code, is you might be routing audio into a queue but your program doesn't remove that data from the queue. That sort of scenario would quickly consume all the audio memory, causing everything in the audio library to stop working.

The queue objects are meant to give access to the raw audio data. But this is a difficult matter, because most high-level Arduino style C++ code uses APIs which can block for unknown time, especially with writing to SD cards. The idea behind queues in the audio library will automatically buffer the audio stream data in the queue, so your program can work with with the raw data in bursts without the strict real-time timing requirement to process each block within its native timing. The queue buffering is supposed to make working with the raw data easier. But if you're not careful to eventually remove data from every active queue before the audio memory you allocated gets used up, the entire audio library will starve for memory and all inputs and synths can't give you anything more because they have no way to access more memory.

TeensyPhonon · Apr 25, 2024

You're right... the mono code uses 9 blocks while the stereo one uses 10 (maximum). So I increased the audio memory to 11, and it works!

Then, I started to become crazy because increasing the audio memory is the first thing I tried. So, I did a test with 12, and it does not work! It seems that it only works when the number of blocks is odd?!

At least, it highlighted that my code seems a little bit wonky...

MarkT · Apr 25, 2024

If the system is starved of blocks odd things can happen if something is freeing blocks at all, don't take the fact it "works" with 11 to mean its properly working, for if it were 12 would definitely work too.

PaulStoffregen · Apr 25, 2024

For the sake of testing, I'd try allocating many more blocks, like 100. Then if you see your usage is in the 90s, you'll know one or more of the queues is hogging memory.

TeensyPhonon · Apr 25, 2024

I actually tested it with a bunch of values, and it only works if the number of blocks is odd.

I think it comes from the fact that the input is stereo while the output is mono, so it is not possible for the input queues to allocate everything, leaving a spare buffer for the output queue?

Actually, the microphone input did not work either... so I changed the code to begin input queues only when needed and end and clear them when the work is finished. It seems to work correctly (I hope).

PaulStoffregen · Apr 25, 2024

If you want me to investigate, please post another complete program to demonstrate the problem. Even if the change from the code you showed in msg #5 is trivial, I need you to understand I have a long history of wasting time not able to reproduce a problem when the code shown needs changes.

In whatever code you share, I'm specifically looking to see your use of AudioMemoryUsage() to monitor how much memory it is really using. Please make sure you keep that in the code you share here.

TeensyPhonon · Apr 26, 2024

The code I used is :

C:

#include <Audio.h>
#include <arm_const_structs.h>
#include <utility/imxrt_hw.h>
#include <SD.h>
#include <SPI.h>


File                         dataFile;
const int                    chipSelect = BUILTIN_SDCARD;           

extern "C" uint32_t set_arm_clock(uint32_t frequency);
const float32_t PROGMEM audio_gain = 0.05;

const int Npartitions = 500;
const double PROGMEM SAMPLE_RATE = AUDIO_SAMPLE_RATE_EXACT;
const int PROGMEM partitionsize = AUDIO_BLOCK_SAMPLES;

const float32_t T = Npartitions*partitionsize/SAMPLE_RATE;
const int nc = T*SAMPLE_RATE;
const int N_memory = 15; //only works if odd
float32_t mem_delay = N_memory*partitionsize/SAMPLE_RATE;

int16_t *sp_L;
uint8_t first_block = 1;
const int PROGMEM nfor = nc / partitionsize;

float32_t DMAMEM float_buffer_L[nc];

char buffer[4];
float32_t chirp[nc];

AudioInputI2S               i2s_in;
AudioRecordQueue            Q_in_L;
AudioRecordQueue            Q_in_R;
AudioPlayQueue              Q_out_L;
AudioOutputI2S              i2s_out;
AudioConnection             patchCord_in(i2s_in, 0, Q_in_L, 0);
AudioConnection             patchCord_in_R(i2s_in, 1, Q_in_R, 0);
AudioConnection             patchCord_out(Q_out_L, 0,  i2s_out, 0);

String command;


void setup() {

  SD.begin(chipSelect);
 

  AudioMemory(N_memory);
  delay(100);

 
  pinMode(0,INPUT_PULLDOWN);
  pinMode(1,OUTPUT);
  pinMode(2,OUTPUT);
  pinMode(LED_BUILTIN,OUTPUT); 

  setI2SFreq(SAMPLE_RATE);

  digitalWrite(1,HIGH);
  digitalWrite(LED_BUILTIN,HIGH);

  Serial.begin(115200);
  while(!Serial);

  Serial.println(SAMPLE_RATE);
  Serial.println(partitionsize);
  Serial.println(Npartitions);

  //dummy signal for test
  for(int i = 0;i<nc;i++){
    chirp[i] = 0.1*sin(TWO_PI*440*(float)i/SAMPLE_RATE);
  }
 
}

            

void loop() {

  if(Serial.available()){
    
    Serial.println(AudioMemoryUsage());

    command = Serial.readString();

    if(command.startsWith("play")){

      digitalWrite(1,LOW);
      
      for (int i = 0; i < nfor; i++) {
        sp_L = Q_out_L.getBuffer();
        arm_float_to_q15 (&chirp[partitionsize * i], sp_L, partitionsize);
        Q_out_L.playBuffer();
      }

      digitalWrite(1,HIGH);
    }
      
  }

}


void setI2SFreq(int freq) {
  int n1 = 4;
  int n2 = 1 + (24000000 * 27) / (freq * 256 * n1);
  double C = ((double)freq * 256 * n1 * n2) / 24000000;
  int c0 = C;
  int c2 = 10000;
  int c1 = C * c2 - (c0 * c2);
  set_audioClock(c0, c1, c2, true);
  CCM_CS1CDR = (CCM_CS1CDR & ~(CCM_CS1CDR_SAI1_CLK_PRED_MASK | CCM_CS1CDR_SAI1_CLK_PODF_MASK)) | CCM_CS1CDR_SAI1_CLK_PRED(n1-1) | CCM_CS1CDR_SAI1_CLK_PODF(n2-1);
}

If anyone is interested, to make everything fully work, I had to use queues only when needed, like so:

C:

if(...){
    queue.begin();
    queue.clear();

    //code using the queue

    queue.end();
    queue.clear();
}

PaulStoffregen · Apr 26, 2024

TeensyPhonon said:
to make everything fully work, I had to use queues only when needed, like so:

Yes, this is expected behavior. If you leave any record queue active but your program doesn't remove the data, incoming audio quickly uses up the available memory. It is not a bug in the audio library. This is the way queues work.

Two queues on i2s input not working

TeensyPhonon

Active member

h4yn0nnym0u5e

Well-known member

TeensyPhonon

Active member

h4yn0nnym0u5e

Well-known member

TeensyPhonon

Active member

PaulStoffregen

Well-known member

TeensyPhonon

Active member

MarkT

Well-known member

PaulStoffregen

Well-known member

TeensyPhonon

Active member

PaulStoffregen

Well-known member

TeensyPhonon

Active member

PaulStoffregen

Well-known member