I need help with an SdFat ADC DMA ISR example

[Bill Greiman]

I am working on an improved RingBuf ISR example for SdFat. I want to handle the cache for DMAMEM correctly and achieve the maximum ADC rate.

This is an example to test at maximum speed so I am not concerned with accuracy.

It appears that arm_dcache_delete() is only required on Teensy 4.x and only for RAM2. Please look at the example code and see if my use of arm_dcache_delete() in the ISR is correct.

Also look at the ADC setup and see if it gives maximum ADC sample rate.

The example runs at about 1.56 msps or 3.125 MB/sec on Teensy 4.1 and 1.36 msps or 2.727 MB/sec on Teensy 3.6.

I have modified the SdFat RingBuf class to work in an ISR or in a multi-thread system like ESP32.

I have attached the example with a modified version RingBuf.h. It should work with the current version of SdFat.

Here is the example code:

// Test of Teensy exFAT DMA ADC logger.
// This is mainly to test use of RingBuf in an ISR.
// This example only supports pins on the first ADC.
// it has only been tested on Teensy 3.6 and 4.1.
// You should modify it for serious use as a data logger.
//
#include "ADC.h"
#include "DMAChannel.h"
#include "FreeStack.h"
#include "RingBuf.h"
#include "SdFat.h"

// Pin must be on first ADC.
#define ADC_PIN A0

// 400 sector RingBuf - could be larger on Teensy 4.1.
const size_t RING_BUF_SIZE = 400 * 512;

// Preallocate 8GiB file.
const uint64_t PRE_ALLOCATE_SIZE = 8ULL << 30;

// Use FIFO SDIO.
#define SD_CONFIG SdioConfig(FIFO_SDIO)

ADC adc;

DMAChannel dma(true);

SdFs sd;

FsFile file;

// Ping-pong DMA buffer.
DMAMEM static uint16_t __attribute__((aligned(32))) dmaBuf[2][256];

// Count of DMA interrupts.
volatile size_t dmaCount;

// RingBuf for 512 byte sectors.
RingBuf<FsFile, RING_BUF_SIZE> rb;

// Shared between ISR and background.
volatile size_t maxBytesUsed;

// Overrun error for write to RingBuf.
volatile bool overrun;
//------------------------------------------------------------------------------
// ISR for DMA.
static void isr() {
  if (!overrun) {
    // Clear cache for buffer filled by DMA to insure read from DMA memory.
    arm_dcache_delete((void*)dmaBuf[dmaCount & 1], 512);
    // Enable RingBuf functions to be called in ISR.
    rb.beginISR();
    if (rb.write(dmaBuf[dmaCount & 1], 512) == 512) {
      dmaCount++;
      if (rb.bytesUsed() > maxBytesUsed) {
        maxBytesUsed = rb.bytesUsed();
      }
    } else {
      overrun = true;
    }
    // End use of RingBuf functions in ISR.
    rb.endISR();
  }
  dma.clearComplete();
  dma.clearInterrupt();
#if defined(__IMXRT1062__)
  // Handle clear interrupt glitch in Teensy 4.x!
  asm("DSB");
#endif  // defined(__IMXRT1062__)
}
//------------------------------------------------------------------------------
#if defined(__IMXRT1062__)  // Teensy 4.x
#define SOURCE_SADDR ADC1_R0
#define SOURCE_EVENT DMAMUX_SOURCE_ADC1
#else
#define SOURCE_SADDR ADC0_RA
#define SOURCE_EVENT DMAMUX_SOURCE_ADC0
#endif
//------------------------------------------------------------------------------
static void init(uint8_t pin) {
  dma.begin();
  dma.attachInterrupt(isr);
  dma.source((volatile const signed short&)SOURCE_SADDR);
  dma.destinationBuffer((volatile uint16_t*)dmaBuf, sizeof(dmaBuf));
  dma.interruptAtHalf();
  dma.interruptAtCompletion();
  dma.triggerAtHardwareEvent(SOURCE_EVENT);
  dma.enable();
  adc.adc0->enableDMA();
  adc.adc0->startContinuous(pin);
}
//------------------------------------------------------------------------------
void stopDma() {
  adc.adc0->disableDMA();
  dma.disable();
}
//------------------------------------------------------------------------------
void printTest(Print* pr) {
  if (file.fileSize() < 1024 * 2) {
    return;
  }
  file.rewind();
  rb.begin(&file);
  // Could readIn RING_BUF_SIZE bytes and write to a csv file in a loop.
  if (rb.readIn(2048) != 2048) {
    sd.errorHalt("rb.readIn failed");
  }
  uint16_t data;
  for (size_t i = 0; i < 1024; i++) {
    pr->print(i);
    pr->print(',');
    // Test read with: template <typename Type>bool read(Type* data).
    rb.read(&data);
    pr->println(data);
  }
}
//------------------------------------------------------------------------------
void runTest(uint8_t pin) {
  dmaCount = 0;
  maxBytesUsed = 0;
  overrun = false;
  do {
    delay(10);
  } while (Serial.read() >= 0);

  if (!file.open("IsrLoggerTest.bin", O_CREAT | O_TRUNC | O_RDWR)) {
    sd.errorHalt("file.open failed");
  }
  if (!file.preAllocate(PRE_ALLOCATE_SIZE)) {
    sd.errorHalt("file.preAllocate failed");
  }
  rb.begin(&file);
  Serial.println("Type any character to stop\n");

  init(pin);
  uint32_t samplingTime = micros();
  while (!overrun && !Serial.available()) {
    size_t n = rb.bytesUsed();
    if ((n + file.curPosition()) >= (PRE_ALLOCATE_SIZE - 512)) {
      Serial.println("File full - stopping");
      break;
    }
    if (n >= 512) {
      if (rb.writeOut(512) != 512) {
        Serial.println("writeOut() failed");
        file.close();
        return;
      }
    }
  }
  stopDma();
  samplingTime = micros() - samplingTime;
  if (!rb.sync()) {
    Serial.println("sync() failed");
    file.close();
    return;
  }
  if (!file.truncate()) {
    sd.errorHalt("truncate failed");
  }
  if (overrun) {
    Serial.println("Overrun ERROR!!");
  }
  Serial.print("dmsCount ");
  Serial.println(dmaCount);
  Serial.print("RingBufSize ");
  Serial.println(RING_BUF_SIZE);
  Serial.print("maxBytesUsed ");
  Serial.println(maxBytesUsed);
  Serial.print("fileSize ");
  file.printFileSize(&Serial);
  Serial.println();
  Serial.print(0.000001 * samplingTime);
  Serial.println(" seconds");
  Serial.print(1.0 * file.fileSize() / samplingTime, 3);
  Serial.println(" MB/sec\n");
  printTest(&Serial);
  file.close();
}
//------------------------------------------------------------------------------
void waitSerial(const char* msg) {
  do {
    delay(10);
  } while (Serial.read() >= 0);
  Serial.println(msg);
  while (!Serial.available()) {
  }
  Serial.println();
}
//------------------------------------------------------------------------------
void setup() {
  Serial.begin(9600);
  while (!Serial) {
    yield();
  }
  waitSerial("Type any character to begin");
  Serial.print("FreeStack: ");
  Serial.println(FreeStack());
}
//------------------------------------------------------------------------------
void loop() {
  if (!sd.begin(SD_CONFIG)) {
    sd.initErrorHalt(&Serial);
  }
  // Try for max speed.
  adc.adc0->setAveraging(1);
  adc.adc0->setResolution(10);
  adc.adc0->setConversionSpeed(ADC_CONVERSION_SPEED::VERY_HIGH_SPEED);
  adc.adc0->setSamplingSpeed(ADC_SAMPLING_SPEED::VERY_HIGH_SPEED);
  runTest(ADC_PIN);
  waitSerial("Type any character to run test again");
}

Here is a chart of Teensy 4.1 data:

 

[KurtE]

Hi Bill,

Will try to look through it more soon.
arm_dcache_delete - is potentially needed anywhere that the hardware cache is used. It is not used on DTCM. As you mentioned it is needed for RAM2, and also
if you are using a Teensy 4.1 that uses external memory (optional chips soldered onto the bottom of the T4.1)

An interesting thing about the arm_dcache_delete and the other arm_dcache... commands is it works on blocks of 32 bytes of memory. So the call to this may impact memory before and after the block you passed in.

Which can cause interesting issues, as this call deletes the stuff from the cache without first writing the contents out Which cause some subtle issues.
like suppose you malloc 16 bytes, and then use this call to clear out a cache. That call might cause the heap data structures to be corrupted if, they end up residing in the range of
memory that the delete call hits.

Which is why sometimes I simply call arm_dcache_flush_delete instead as it will flush everything first... FrankB also had a Safe version of the delete that checks if the first block and/or last block are partial blocks and does a flush on those sections.

 

[rjsdotorg]

I am working on an improved RingBuf ISR example for SdFat. I want to handle the cache for DMAMEM correctly and achieve the maximum ADC rate.

This is an example to test at maximum speed so I am not concerned with accuracy.

In addendum to another post, here is my modified version of RingBuf ISR example (based on TeensyDmaAdcLogger.ino)

Output:
FreeStack: 243912
dmaCount 4577
RingBufSize 204800
maxBytesUsed 29696
fileSize 2343.424kB
1.0000 seconds
1171.712000 ksamp/sec

It always does 512 byte writes, but I can't follow in the lib to see if it does:
- always test for SD.is_busy()
- write only if SD is NOT BUSY