Forum Rule: Always post complete source code & details to reproduce any issue!
Results 1 to 10 of 10

Thread: Fast CANBus Logger

  1. #1
    Senior Member
    Join Date
    Jan 2015
    Location
    France
    Posts
    174

    Fast CANBus Logger

    Hi,

    I'm currently try to achieve a CANBus logger. This will become a "black box" on some rally cars. But I am not a programmer :-(

    While Arduino look simple and Teensy fast, I think I haven't the good approach for this project. I have a simple code that read data from CAN and parse them to an SDcard. I fork this code from a project that belong to Pawelsky (thanks to him). The problem here is that I miss some CAN frames. In fact I don't miss CAN reads, all frames can be read by Teensy. I miss some frame because of the SD write. It's not a big miss, around 1%.

    Code:
    /* 
      Format des LOG (16 bytes par message CAN): TS TS TS TS CI CI LE MA DA DA DA DA DA DA DA DA
      TS - horodatage           (4 byte - un nombre - little endian - exprimé en microsecondes)
      CI - ID CAN               (2 byte - un nombre - little endian)
      LE - longueur de la trame (1 byte - un nombre)
      MA - Marqueur             (1 byte - FF en mode normal, passe à 00 si action sur bouton de marquage des LOG)
      DA - données              (8 bytes)
    */
    
    #include <FlexCAN.h>
    #include <SdFat.h>
    #include <Time.h>
    #include <TimeLib.h>
    
    #define BOUTON 20
    #define LED 23
    const unsigned long debouncing_time = 50;
    volatile unsigned long last_millis;
    
    // Version du firmware
    char VERSION[15] = "CANLOGGER v1.5";
    
    // Défini la pin ChipSelect de la carte SD
    const uint8_t SD_CHIP_SELECT = 10;
    
    // Paramètre la vitesse du CAN
    uint32_t VitesseCAN = 500000;
    FlexCAN CANbus(VitesseCAN);
    
    
    // structure canMsg qui recoie la trame CAN
    CAN_message_t canMsg;
    
    // Variables de temps
    uint32_t TempsActuel = 0;
    uint32_t TempSynchro = 5000000;
    
    // Variable compteur de trames
    uint32_t nb_trames = 0;
    
    // Flux de sortie sur le port série
    ArduinoOutStream cout(Serial);
    
    // Caractère pour traitement port COM
    char c;
    
    // Varibles pour la création du nom de fichier sur la carte SD
    char fileName[24];
    
    // Structure "entete" => format d'une ligne a écrire dans le fichier .LOG
    union
    {
      struct  __attribute__((packed))
      {
        uint32_t timestamp;
        uint16_t canId;
        uint8_t  dataLen;
        uint8_t  marquage;
      } header;
      uint8_t      bytes[8];
    } entete;
    
    // Stocke les messages d'erreur en Flash pour économiser de la RAM
    #define error(s) sd.errorHalt(F(s))
    
    // Nomme les instances de SDfat pour le traitement des fichiers
    SdFat sd;
    SdFile file;
    Sd2Card card;
    
    // Lie la bibliothèque Time.h au Hard Teensy
    time_t getTeensy3Time()
    {
      return Teensy3Clock.get();
    }
    
    /*
     * *****************************************************************************************************************************
     * ***           Initialisation - Cette procédure VOID SETUP est éxecuté à chaque mise sous tension du teensy                ***
     * *****************************************************************************************************************************
    */
    void setup()
    {
      // Affecte la PIN à la LED d'etat de fonctionnement
      pinMode(LED, OUTPUT);
    
      // Affecte la PIN au bouton de marquage des LOG
      pinMode(BOUTON, INPUT);
      digitalWrite(BOUTON, HIGH);
      attachInterrupt(BOUTON, ISR_debounce_BOUTON, FALLING);
    
      // Affecte l'horloge RTC du teensy à la librairie Time
      setSyncProvider(getTeensy3Time);
    
      // Initialise le port COM
      Serial.begin (115200); //Le port USB est toujours à 12 Mb/s
      delay (1000);
    
      // Vérifie que l'horloge RTC est bien utilisée
      if (timeStatus() != timeSet) {
        cout << F("Impossible de synchroniser l'horodatage avec l'horloge RTC");
      } else {
        cout << F("L'horodatage a correctement ete synchronisee avec l'horloge RTC");
      }
    
      // Crée le nom complet du fichier LOG avec horodatage
      sprintf(fileName, "%02d-%02d-%04d_%02d.%02d.%02d.tcl", day(), month(), year(), hour(), minute(), second());
    
      // Initialise la communication avec la carte SD par liaison SPI
      // 10 est la pin du Teensy pour le Chip Select (pin digitale)
      if (!sd.begin(SD_CHIP_SELECT, SPI_FULL_SPEED)) {
        sd.initErrorHalt();
      }
    
      // Création du fichier LOG
      SdFile::dateTimeCallback(dateTime);
      if (!file.open(fileName, O_CREAT | O_APPEND | O_WRITE)) {
        error("Ouverture du fichier impossible");
      }
    
      // Assigne FF à la valeur de marqueur
      entete.header.marquage = 0xFF;
    
      // Initialise le bus CAN
      CANbus.begin();
    }
    
    
    /* *****************************************************************************************************************************
     * ***                    Boucle Principale - La procédure VOID LOOP est le main pour les Arduino                            ***
     * *****************************************************************************************************************************
    */
    
    void loop()
    {
      // micros() représente le temps en microsecondes depuis lequel le teensy est sous tension - permet de créer un repère
      // temporel dans le fichier LOG pour les trames recues. Sur un Teensy micros() a une résolution de 2 µs
      TempsActuel = micros();
    
      // Vérifie si des données sont disponibles sur le bus CAN
      if (CANbus.available())
      {
        CANbus.read(canMsg);
        entete.header.timestamp = (uint32_t)TempsActuel;
        entete.header.canId = (uint16_t)canMsg.id;
        entete.header.dataLen = (uint8_t)canMsg.len;
        file.write(entete.bytes, 8);
        file.write(canMsg.buf, 8);
        nb_trames ++;
      }
    
      // Est-il est temps d'écrire sur la carte SD ?
      if (TempSynchro < TempsActuel)
      {
        TempSynchro = TempsActuel + 5000000;
        file.sync();
      }
    
      // Avons nous recu une commande sur le port COM ?
      if (Serial.available())
      {
        Traiter_Commande();
      }
    }
    I discuss about that with a friend and he tell me that I need FIFO Queues. I'm quite sure there are already Queues in CAN library, I'm sure that there are some in SDFat too, but I'm unsure how to use them. I'm also thinking that DMA can help.

    SDFat have a "lowlatencylogger" example that could help, but I'm not sure this can solve the issue because this example use a temp file and write the final one when user stop logging.

    My logger should start at car key-on and stop at car key-off AND log file should be usable as is. I can ad a little battery t the logger if needed to allow the file close, but I prefer the file append. I can loose the last "buffer" or say the last SDcard cluster without any problem, but I really like a 0%miss log file.

    also, at the end of the sketch there is code that look at serial COM for a possible command, which will only append when user want to download files from SDcard to PC. Maybe shall I use an interrupt here ?

    Any advice is welcome.
    Regards,
    Manu
    Last edited by Manu; 07-12-2016 at 08:21 AM. Reason: Clean code sample

  2. #2
    Senior Member
    Join Date
    Feb 2016
    Location
    Australia
    Posts
    282
    Not a programmer...
    But when I tested my teensy 3.2 with sd card adapter I got 18ms for small writes.

    I just created an array to store a few hundred bytes and write the array when its full.

    Can you provide a link to pawelsky's project.

    Thanks.
    Last edited by Gibbedy; 07-17-2016 at 06:43 PM.

  3. #3
    Senior Member Constantin's Avatar
    Join Date
    Nov 2012
    Location
    In the yard with a 17' Dia. Ferris Wheel
    Posts
    1,408
    I ran into a similar issue with my power logger (1.4ksps). My crude solution: two CPUs. One to do the readings, conversions, etc. and the other to deal with the SDHC card. They communicate via Serial using EasyTransfer. The gatherer CPU gathers and processes, then sends bursts of packets during idle time to the SD CPU. Hence, no impact re latency and so on.

    I bet that the magicians here can do the same thing with one CPU by using Interrupts/DMA/Can Bus buffers effectively but this hardware solution works for me.

  4. #4
    Senior Member
    Join Date
    Jan 2013
    Posts
    843
    The Teensy CAN hardware has support for 6 RX messages. So it will overflow rather quickly. The hardware can trigger interrupts when messages are received and you could modify the flexcan library for that. Then you could copy the messages to a much larger RAM buffer in the interrupt handler.

    Chapter 44 has programming information for the CAN hardware:
    https://www.pjrc.com/teensy/K20P64M72SF1RM.pdf

    The only way to get sane latencies with SD cards is to use pre-allocated files with pre-erased data blocks along the lines of lowlatencylogger. Otherwise you will encounter latency spikes of hundreds of milliseconds.

  5. #5
    Senior Member Constantin's Avatar
    Join Date
    Nov 2012
    Location
    In the yard with a 17' Dia. Ferris Wheel
    Posts
    1,408
    Hence my utilization of a dedicated SD card CPU. The delays are simply too unpredictable, as they seem to depend on (in part) to the card used, the local weather, star constellations, and other factors outside my control. I'm just a sub mechE, so diving into interrupts is avoided as much as possible. I only use them in my code for silly stuff like counters.

    Sure, Serial and so on use interrupts to function, but that's outside my code base. Others, much smarter than me, made those work beautifully. Shoulders of giants and all that.

  6. #6
    Senior Member
    Join Date
    Jan 2015
    Location
    France
    Posts
    174
    Quote Originally Posted by Gibbedy View Post
    Can you provide a link to pawelsky's project.
    https://oshpark.com/shared_projects/VeJFD9qA

  7. #7
    Senior Member
    Join Date
    Jan 2015
    Location
    France
    Posts
    174
    I would like to thanks all of you for your replies. Many thanks to Constantin for his solution that I won't imagine by myself. I will look at this.

    Just a quick update : with the actual code, there is a 0.1% miss when I send 2000 frames per second with a 500kbaud CANbus (about 50% duty). It look reasonable and I'm sure that it can be optimized and achievable with a single processor and maybe a battery to allow teensy to close the file when the car is keyed off.

    Regards,
    Manu

  8. #8
    very interesting, do you have also a program that replay the logs to can?

  9. #9
    Junior Member
    Join Date
    Feb 2017
    Posts
    1
    Hi guys,
    just came across and in addition to pawelsky's project, want to point to my own project of CAN logger [tindie , ebay]

  10. #10
    Junior Member
    Join Date
    Oct 2015
    Posts
    17
    Manu, the trick is to figure out that "LowLatencyLogger". It actually writes to the SD card in blocks permanently. However, you need to write a parsing script of some sort that can parse the binary data and throw out the junk data at the end of the file if/when the car gets turned off. This should give you ~ 200 microsecond write times for 512 byte blocks. This should be enough, if you've gone ahead and cleared out the CAN RX buffers. For an example, see the code snippets below:


    Defining the struct of data:
    Code:
    // Salus_Logging.h
    
    #pragma once
    
    #include "Salus_Common.h"
    #include <SystemInclude.h>
    #include <SdFatUtil.h>
    #include <SdFatConfig.h>
    #include <SdFat.h>
    #include <MinimumSerial.h>
    
    typedef struct __attribute__ ((packed)) {
    
        // Floats are 32-bit on Teensy 3.x with Teensyduino 1.6.x
        // Doubles are 64-bit on Teensy 3.x with Teensyduino 1.6.x
    
        uint8_t dataVersion = 2;
        
        uint32_t clock;
        
        // GPS data
        uint8_t hour;
        uint8_t minute;
        uint8_t seconds;
        uint8_t satellites;
    
        float latitude;
        float longitude;
        float gpsSpeed;
        float gpsAltitude;
    
        // Barometer data
        float pressure;
        float altitude;
        float temperature;
    
        // ADXL Data
        float adxlX;
        float adxlY;
        float adxlZ;
    
        // IMU Data
        float bnoAx;
        float bnoAy;
        float bnoAz;
        float bnoGx;
        float bnoGy;
        float bnoGz;
    
        double quatW;
        double quatX;
        double quatY;
        double quatZ;
    
    } salus_data_t;
    
    salus_data_t* getBuffer();
    
    void writeData();
    void writeHeader();
    void startLogger();
    void loggingTask();
    
    void startBinLogger(void (*dateTime)(uint16_t*,uint16_t*));
    void fastLog();
    Functions to create the file and store the data:
    Code:
    // 
    // 
    // 
    
    #include "Salus_Logging.h"
    
    #define error(s)            (sd.errorHalt(F(s)))
    #define SD_CHIPSELECT       (17)
    #define SD_SPI_SPEED        (SPI_FULL_SPEED)
    
    SdFat sd;
    SdFile file;
    SdBaseFile binFile;
    
    // Number of data records in a block.
    const uint16_t DATA_DIM = 512 / sizeof(salus_data_t);
    
    //Compute fill so block size is 512 bytes.  FILL_DIM may be zero.
    const uint16_t FILL_DIM = 512 - DATA_DIM*sizeof(salus_data_t);
    
    // Maximum file size in blocks.
    // The program creates a contiguous file with FILE_BLOCK_COUNT 512 byte blocks.
    // This file is flash erased using special SD commands.
    // 360000 entries is good for 3600 seconds of logging (60 minutes)
    const uint32_t FILE_BLOCK_COUNT = (360000 / DATA_DIM);
    
    // max number of blocks to erase per erase call
    uint32_t const ERASE_SIZE = 262144L;
    
    uint32_t bgnBlock, endBlock, blockNum = 0;
    
    struct block_t {
        salus_data_t data[DATA_DIM];
        uint8_t fill[FILL_DIM];
    };
    
    block_t block;
    
    void fastLog(){
        if (blockNum == DATA_DIM-1){
            if (!sd.card()->isBusy()){
                if (!sd.card()->writeData((uint8_t*)&block)){
                    error("fast write failed");
                }
                blockNum = 0;
            }
            else
                Serial.println("Card BUSY!");
        }
        else
            blockNum++;
    }
    
    void startBinLogger(void (*dateTime)(uint16_t *date, uint16_t *time)){
    
    #ifdef LOGGER_DEBUG
        Serial.print("Size of Struct: ");
        Serial.println(sizeof(salus_data_t));
        Serial.print("Data_DIM: ");
        Serial.println(DATA_DIM);
        Serial.print("FILL_DIM: ");
        Serial.println(FILL_DIM);
        Serial.print("Sizeof Block: ");
        Serial.println(sizeof(block_t));
        Serial.println();
    #endif
    
        if (!sd.begin(SD_CHIPSELECT, SD_SPI_SPEED)) {
            sd.initErrorHalt();
        }
    
        int number = 0;
        char sName[80];
    
        // Find a filename that hasn't been used already
        do
        {
            sprintf(sName, "Salus_Results_%d.bin", number++);
        } while (sd.exists(sName));
    
        binFile.close();
    
        binFile.dateTimeCallback(dateTime);
    
        if (!binFile.createContiguous(sd.vwd(), sName, 512 * FILE_BLOCK_COUNT)){
            error("createContiguous failed");
        }
    
        if (!binFile.contiguousRange(&bgnBlock, &endBlock)){
            error("contiguousRange failed");
        }
    
        // Use SdFat's internal buffer ( ???? )
        uint8_t* cache = (uint8_t*)sd.vol()->cacheClear();
        if (cache == 0) {
            error("cacheClear failed");
        }
    
        binFile.dateTimeCallbackCancel();
    
        uint32_t bgnErase = bgnBlock;
        uint32_t endErase;
        while (bgnErase < endBlock) {
            endErase = bgnErase + ERASE_SIZE;
            if (endErase > endBlock) {
                endErase = endBlock;
            }
            if (!sd.card()->erase(bgnErase, endErase)) {
                error("erase failed");
            }
            bgnErase = endErase + 1;
        }
    
    
        // Start a multiple block write.
        if (!sd.card()->writeStart(bgnBlock, FILE_BLOCK_COUNT)) {
            error("writeBegin failed");
        }
    }
    
    salus_data_t* getBuffer(){
        return &(block.data[blockNum]);
    }
    Python script to parse the data:
    Code:
    import struct
    import os
    
        # // Floats are 32-bit on Teensy 3.x with Teensyduino 1.6.x
        # // Doubles are 64-bit on Teensy 3.x with Teensyduino 1.6.x
    
        # uint8_t dataVersion = 2;
        
        # uint32_t clock;
        
        # // GPS data
        # uint8_t hour;
        # uint8_t minute;
        # uint8_t seconds;
        # uint8_t satellites;
    
        # float latitude;
        # float longitude;
        # float gpsSpeed;
        # float gpsAltitude;
    
        # // Barometer data
        # float pressure;
        # float altitude;
        # float temperature;
    
        # // ADXL Data
        # float adxlX;
        # float adxlY;
        # float adxlZ;
    
        # // IMU Data
        # float bnoAx;
        # float bnoAy;
        # float bnoAz;
        # float bnoGx;
        # float bnoGy;
        # float bnoGz;
    
        # double quatW;
        # double quatX;
        # double quatY;
        # double quatZ;
    
    keys = (
        'dataVersion', 'milliseconds',
        'hour', 'minute', 'seconds',
        'satellites', 'latitude', 'longitude',
        'gpsSpeed', 'gpsAltitude',
        'pressure', 'AGL', 'temperature',
        'adxlX', 'adxlY', 'adxlZ',
        'imuAx', 'imuAy', 'imuAz',
        'imuGx', 'imuGy', 'imuGz',
        'quatW', 'quatX', 'quatY', 'quatZ',
    )
    structFormat = "<BI4B16f4d"
    dataSize = 512//struct.calcsize(structFormat)*struct.calcsize(structFormat)
    
    for filename in os.listdir():
        if ".bin" in filename:
            dataEntries = []
            fileSize = os.stat(filename).st_size
            with open(filename, "rb") as inFile:
                for x in range(fileSize//512):
                    for entry in struct.iter_unpack(structFormat, inFile.read(dataSize)):
                        if not(entry[0] == 0 or entry[0] == 255):
                            dataEntries.append(entry)
                    inFile.read(512-dataSize)
    
            import csv
            with open(filename.replace(".bin",".csv"),'w', newline='') as csvfile:
                csvWriter = csv.writer(csvfile, dialect='excel')
                csvWriter.writerow(keys)
                for entry in dataEntries:
                    csvWriter.writerow(entry)

Posting Permissions

  • You may not post new threads
  • You may not post replies
  • You may not post attachments
  • You may not edit your posts
  •