Hello everyone,
I'm developing a datalogger and I experience a couple of problems.
To start with, here is my code:
The original code is the Lowlatency logger from SDfat from Greiman.
The logger logs CANData. Originally the LowLatency logger logged sensor values at a regular interval, my logger will log a CAN bus (based on incoming messages, not interval) with a max of 600-700 messages per seconds top. The logger works fine for my needs except for the following:
- Last filename is not being renamed properly (doesn't give an error when I truncate the file). So for instance, if I would start the log 3 times (filename is based on date) it will show 150300.bin 150301.bin and tmp_log.bin. Somehow renaming the file does not actually happen, but as soon you start logging new data the previous file is actually been renamed. So if I would start logging 4 times it would have 150300 150301 150302 tmp_log.bin, very strange. Hope this fuzzy explanations helps.
- Writing header file works, but is there an easier way to do this? My solution is kind of basic, I cloned the acquiredata function and insert different data (not a CAN message, but time + loggerid + ....)
Croz8
I'm developing a datalogger and I experience a couple of problems.
To start with, here is my code:
Code:
/* sd.vwd() = volume working directory (initially this is root)
*
* SDFat by Greiman
*/
#include <Time.h>
#include <TimeLib.h>
#include <SPI.h>
#include <SdFat.h>
#include <elapsedMillis.h>
#include <SdFatUtil.h>
#include <FlexCAN.h> //FLEXCAN LIBRARY
#include <kinetis_flexcan.h>
// CONFIG FOR LOGGER [Eventually external on µSd]
FlexCAN CANbus(125000); //125/s
int versionv[8] = {86,49,46,48,48,69,81,77};
static int loggerID = 3001;
// -----------------------------------------------------------------------------------------------
static CAN_message_t rxmsg;
#include "UserDataType.h" // Edit this include file to change data_t.
int incomingChar = 0;
elapsedMillis timeElapsed; //GLOBAL FOR INTERRUPT 1PPPS
// Acquire a data record.
void acquireData(data_t* data) {
data->id = rxmsg.id;
data->time_s = timeElapsed;
for (int i = 0; i < ADC_DIM; i++) {
data->buf[i] = rxmsg.buf[i];
}
}
void printHeads(data_t* data) {
updatedate();
data->id = loggerID;
data->time_s = now();
timeElapsed = 0;// Start at a interval.
for (int i = 0; i < 8; i++) {
data->buf[i] = versionv[i];
}
}
// Print a data record.
void printData(Print* pr, data_t* data) {
pr->print(data->id);
pr->write(',');
pr->print(data->time_s);
for (int i = 0; i < ADC_DIM; i++) {
pr->write(',');
pr->print(data->buf[i]);
}
pr->println();
}
// Print data header.
void printHeader(Print* pr) {
pr->print(F("MSG id"));
pr->print(F(",Time passed"));
for (int i = 0; i < ADC_DIM; i++) {
pr->print(F(",byte"));
pr->print(i,HEX);
}
pr->println();
}
void updatedate(){
setTime(1458064936); //IMPLEMENT TIME (eventually GPS will import rtc)
}
//==============================================================================
// Start of configuration constants.
//==============================================================================
//Interval between data records in microseconds.
//------------------------------------------------------------------------------
// Pin definitions.
//
// SD chip select pin.
const uint8_t SD_CS_PIN = 10;
//
// Digital pin to indicate an error, set to -1 if not used.
// The led blinks for fatal errors. The led goes on solid for SD write
// overrun errors and logging continues.
const int8_t ERROR_LED_PIN = -1;
//------------------------------------------------------------------------------
// File definitions.
//
// Maximum file size in blocks.
// The program creates a contiguous file with FILE_BLOCK_COUNT 512 byte blocks.
// This file is flash erased using special SD commands. The file will be
// truncated if logging is stopped early.
const uint32_t FILE_BLOCK_COUNT = 1024000;
// log file base name. Must be six characters or less.
#define FILE_BASE_NAME "0000"
//------------------------------------------------------------------------------
// Buffer definitions.
//
// The logger will use SdFat's buffer plus BUFFER_BLOCK_COUNT additional
// buffers.
//
const uint8_t BUFFER_BLOCK_COUNT = 18;
//==============================================================================
// End of configuration constants.
//==============================================================================
// Temporary log file. Will be deleted if a reset or power failure occurs.
#define TMP_FILE_NAME "tmp_log.bin"
// Size of file base name. Must not be larger than six.
const uint8_t BASE_NAME_SIZE = sizeof(FILE_BASE_NAME) - 1;
SdFat sd;
SdBaseFile binFile;
char binName[13] = FILE_BASE_NAME "00.bin";
// Number of data records in a block.
const uint16_t DATA_DIM = (512 - 4) / sizeof(data_t);
//Compute fill so block size is 512 bytes. FILL_DIM may be zero.
const uint16_t FILL_DIM = 512 - 4 - DATA_DIM * sizeof(data_t);
struct block_t {
uint16_t count;
uint16_t overrun;
data_t data[DATA_DIM];
uint8_t fill[FILL_DIM];
};
const uint8_t QUEUE_DIM = BUFFER_BLOCK_COUNT + 2;
block_t* emptyQueue[QUEUE_DIM];
uint8_t emptyHead;
uint8_t emptyTail;
block_t* fullQueue[QUEUE_DIM];
uint8_t fullHead;
uint8_t fullTail;
// Advance queue index.
inline uint8_t queueNext(uint8_t ht) {
return ht < (QUEUE_DIM - 1) ? ht + 1 : 0;
}
//==============================================================================
// Error messages stored in flash.
#define error(msg) errorFlash(F(msg))
//------------------------------------------------------------------------------
void errorFlash(const __FlashStringHelper* msg) {
sd.errorPrint(msg);
}
//------------------------------------------------------------------------------
//------------------------------------------------------------------------------
// log data
// max number of blocks to erase per erase call
uint32_t const ERASE_SIZE = 262144L;
void logData() {
uint32_t bgnBlock, endBlock;
// Allocate extra buffer space.
block_t block[BUFFER_BLOCK_COUNT];
block_t* curBlock = 0;
Serial.println();
// Find unused file name.
if (BASE_NAME_SIZE > 6) {
error("FILE_BASE_NAME too long");
}
sprintf(binName, "%02d%02d00.bin", day(), month());
while (sd.exists(binName)) {
if (binName[BASE_NAME_SIZE + 1] != '9') {
binName[BASE_NAME_SIZE + 1]++;
} else {
binName[BASE_NAME_SIZE + 1] = '0';
if (binName[BASE_NAME_SIZE] == '9') {
error("Can't create file name");
}
binName[BASE_NAME_SIZE]++;
}
}
// Delete old tmp file.
if (sd.exists(TMP_FILE_NAME)) {
Serial.println(F("Deleting tmp file"));
if (!sd.remove(TMP_FILE_NAME)) {
error("Can't remove tmp file");
}
}
// Create new file.
Serial.println(F("Creating new file"));
binFile.close();
if (!binFile.createContiguous(sd.vwd(),
TMP_FILE_NAME, 512 * FILE_BLOCK_COUNT)) {
error("createContiguous failed");
}
// Get the address of the file on the SD.
if (!binFile.contiguousRange(&bgnBlock, &endBlock)) {
error("contiguousRange failed");
}
// Use SdFat's internal buffer.
uint8_t* cache = (uint8_t*)sd.vol()->cacheClear();
if (cache == 0) {
error("cacheClear failed");
}
// Flash erase all data in the file.
Serial.println(F("Erasing all data"));
uint32_t bgnErase = bgnBlock;
uint32_t endErase;
while (bgnErase < endBlock) {
endErase = bgnErase + ERASE_SIZE;
if (endErase > endBlock) {
endErase = endBlock;
}
if (!sd.card()->erase(bgnErase, endErase)) {
error("erase failed");
}
bgnErase = endErase + 1;
}
// Start a multiple block write.
if (!sd.card()->writeStart(bgnBlock, FILE_BLOCK_COUNT)) {
error("writeBegin failed");
}
// Initialize queues.
emptyHead = emptyTail = 0;
fullHead = fullTail = 0;
// Use SdFat buffer for one block.
emptyQueue[emptyHead] = (block_t*)cache;
emptyHead = queueNext(emptyHead);
// Put rest of buffers in the empty queue.
for (uint8_t i = 0; i < BUFFER_BLOCK_COUNT; i++) {
emptyQueue[emptyHead] = &block[i];
emptyHead = queueNext(emptyHead);
}
Serial.println(F("Logging - type an S to stop logging"));
// Wait for Serial Idle.
Serial.flush();
delay(10);
uint32_t bn = 0;
uint32_t t0 = millis();
uint32_t t1 = t0;
uint32_t overrun = 0;
uint32_t overrunTotal = 0;
uint32_t count = 0;
uint32_t maxLatency = 0;
bool printHeader = true;
bool closeFile = false;
while (1) {
// Time for next data record.
if (Serial.available()) {
closeFile = true;
}
if (closeFile) {
if (curBlock != 0 && curBlock->count >= 0) {
// Put buffer in full queue.
fullQueue[fullHead] = curBlock;
fullHead = queueNext(fullHead);
curBlock = 0;
}
} else {
if (curBlock == 0 && emptyTail != emptyHead) {
curBlock = emptyQueue[emptyTail];
emptyTail = queueNext(emptyTail);
curBlock->count = 0;
curBlock->overrun = overrun;
overrun = 0;
}
if (curBlock == 0) {
overrun++;
} else {
if(printHeader){
printHeads(&curBlock->data[curBlock->count++]);
printHeader = false;
}
if (CANbus.read(rxmsg)) {
acquireData(&curBlock->data[curBlock->count++]);
if (curBlock->count == DATA_DIM) {
fullQueue[fullHead] = curBlock;
fullHead = queueNext(fullHead);
curBlock = 0;
}
}
}
}
if (fullHead == fullTail) {
// Exit loop if done.
if (closeFile) {
break;
}
} else if (!sd.card()->isBusy()) {
// Get address of block to write.
block_t* pBlock = fullQueue[fullTail];
fullTail = queueNext(fullTail);
// Write block to SD.
uint32_t usec = micros();
if (!sd.card()->writeData((uint8_t*)pBlock)) {
error("write data failed");
}
usec = micros() - usec;
t1 = millis();
if (usec > maxLatency) {
maxLatency = usec;
}
count += pBlock->count;
// Add overruns and possibly light LED.
if (pBlock->overrun) {
overrunTotal += pBlock->overrun;
if (ERROR_LED_PIN >= 0) {
digitalWrite(ERROR_LED_PIN, HIGH);
}
}
// Move block to empty queue.
emptyQueue[emptyHead] = pBlock;
emptyHead = queueNext(emptyHead);
bn++;
if (bn == FILE_BLOCK_COUNT) {
// File full so stop
break;
}
}
}
if (!sd.card()->writeStop()) {
error("writeStop failed");
}
// Truncate file if recording stopped early.
if (bn != FILE_BLOCK_COUNT) {
Serial.println(F("Truncating file"));
if (!binFile.truncate(512L * bn)) {
error("Can't truncate file");
}
}
if (!binFile.rename(sd.vwd(), binName)) {
error("Can't rename file");
}
Serial.print(F("File renamed: "));
Serial.println(binName);
Serial.print(F("Max block write usec: "));
Serial.println(maxLatency);
Serial.print(F("Record time sec: "));
Serial.println(0.001*(t1 - t0), 3);
Serial.print(F("Sample count: "));
Serial.println(count);
Serial.print(F("Samples/sec: "));
Serial.println((1000.0)*count/(t1-t0));
Serial.print(F("Overruns: "));
Serial.println(overrunTotal);
Serial.println(F("Done"));
}
//------------------------------------------------------------------------------
// Convert binary file to csv file.
void binaryToCsv() {
uint8_t lastPct = 0;
block_t block;
uint32_t t0 = millis();
uint32_t syncCluster = 0;
SdFile csvFile;
char csvName[13];
if (!binFile.isOpen()) {
Serial.println();
Serial.println(F("No current binary file"));
return;
}
binFile.rewind();
// Create a new csvFile.
strcpy(csvName, binName);
strcpy(&csvName[BASE_NAME_SIZE + 3], "csv");
if (!csvFile.open(csvName, O_WRITE | O_CREAT | O_TRUNC)) {
error("open csvFile failed");
}
Serial.println();
Serial.print(F("Writing: "));
Serial.print(csvName);
Serial.println(F(" - type any character to stop"));
printHeader(&csvFile);
uint32_t tPct = millis();
while (!Serial.available() && binFile.read(&block, 512) == 512) {
uint16_t i;
if (block.count == 0) {
break;
}
if (block.overrun) {
csvFile.print(F("OVERRUN,"));
csvFile.println(block.overrun);
}
for (i = 0; i < block.count; i++) {
printData(&csvFile, &block.data[i]);
}
if (csvFile.curCluster() != syncCluster) {
csvFile.sync();
syncCluster = csvFile.curCluster();
}
if ((millis() - tPct) > 1000) {
uint8_t pct = binFile.curPosition()/(binFile.fileSize()/100);
if (pct != lastPct) {
tPct = millis();
lastPct = pct;
Serial.print(pct, DEC);
Serial.println('%');
}
}
if (Serial.available()) {
break;
}
}
csvFile.close();
Serial.print(F("Done: "));
Serial.print(0.001*(millis() - t0));
Serial.println(F(" Seconds"));
}
void setup(void) {
Serial.begin(9600);
CANbus.begin();
while (!Serial) {}
if (sizeof(block_t) != 512) {
error("Invalid block size");
}
// initialize file system.
if (!sd.begin(SD_CS_PIN, SPI_FULL_SPEED)) {
sd.initErrorPrint();
}
updatedate();
}
//------------------------------------------------------------------------------
void loop(void) {
// discard any input
while (Serial.read() >= 0) {}
Serial.println();
Serial.println(F("type:"));
Serial.println(F("c - convert file to csv"));
Serial.println(F("d - dump data to Serial"));
Serial.println(F("e - overrun error details"));
Serial.println(F("r - record data"));
while (!Serial.available()) {}
char c = tolower(Serial.read());
// Discard extra Serial data.
do {
delay(10);
} while (Serial.read() >= 0);
if (c == 'c') {
binaryToCsv();
} else if (c == 'd') {
} else if (c == 'e') {
} else if (c == 'r') {
logData();
} else {
Serial.println(F("Invalid entry"));
}
}The original code is the Lowlatency logger from SDfat from Greiman.
The logger logs CANData. Originally the LowLatency logger logged sensor values at a regular interval, my logger will log a CAN bus (based on incoming messages, not interval) with a max of 600-700 messages per seconds top. The logger works fine for my needs except for the following:
- Last filename is not being renamed properly (doesn't give an error when I truncate the file). So for instance, if I would start the log 3 times (filename is based on date) it will show 150300.bin 150301.bin and tmp_log.bin. Somehow renaming the file does not actually happen, but as soon you start logging new data the previous file is actually been renamed. So if I would start logging 4 times it would have 150300 150301 150302 tmp_log.bin, very strange. Hope this fuzzy explanations helps.
- Writing header file works, but is there an easier way to do this? My solution is kind of basic, I cloned the acquiredata function and insert different data (not a CAN message, but time + loggerid + ....)
Croz8
Last edited: