/**************************************************************************
Optimizing an ASCII number file parsing algorithm
MJB 3/26/20
You can enter the following commands in the serial monitor:
's' Create a small file of 160 random numbers in ASCII format
'l' Create a large file of 1 million numbers in ASCII format
'r' Read the ASCII file you have created
'd' Show a disk file directory
The program uses the EXFAT file system, which is part of Bill Greiman's
SDFAT Beta library. I use it because it can handle SD cards > 32GB.
When you read or write a file, the first 160 numbers are displayed, so you can
verify that the parsed numbers match the original output.
The write algorithm produces random numbers between 0 and 99, with an
occasional very large number thrown in--as shown in the original poster's
sample data. File writing is done number by number and takes about 10 times
longer than reading the data in blocks.
The read function works with a 16KB buffer to minimize file operations.
Instead of calling ATOI(), the input is directly converted to the
number--avoiding copies to buffers, etc. etc.
On a T3.6 at 180MHz, the parser can convert 2.9MB of text to 1 million numbers
in 393 milliseconds. That means that the average time to read and parse a number
is about 0.4 microseconds! Of course, sending that number out through a serial
port is going to slow things down a bit ;-)
*********************************************************************************/
#include "SdFat.h"
#include "sdios.h"
#include "FreeStack.h"
#include "ExFatlib\ExFatLib.h"
#include <time.h>
#include <TimeLib.h>
SdExFat sd;
ExFile asciiFile;
#define SD_CONFIG SdioConfig(FIFO_SDIO)
// SDCARD_SS_PIN is defined for the built-in SD on some boards.
#ifndef SDCARD_SS_PIN
const uint8_t SD_CS_PIN = SS;
#else // SDCARD_SS_PIN
// Assume built-in SD is used.
const uint8_t SD_CS_PIN = SDCARD_SS_PIN;
#endif // SDCARD_SS_PIN
#define BUFFSIZE 16384
char cbuffer[BUFFSIZE];
char filename[] = "whatever.txt";
// Change this constant to define file size
// when it is less than 500 debug output is printed to Serial
#define NUMTODISPLAY 160
/*****************************************************************************
Read the Teensy RTC and return a time_t (Unix Seconds) value
******************************************************************************/
time_t getTeensy3Time() {
return Teensy3Clock.get();
}
void setup() {
// put your setup code here, to run once:
while (!Serial) {}
Serial.begin(9600);
Serial.println("\nASCII Parsing test");
if (!sd.begin(SD_CONFIG)) {
Serial.println("\nSDIO Card initialization failed.\n");
} else Serial.println("SDIO initialization done.");
if (sd.fatType() == FAT_TYPE_EXFAT) {
Serial.println("Type is exFAT");
} else {
Serial.printf("Type is FAT%d\n", int16_t(sd.fatType()));
}
// set date time callback function so file gets a good date
SdFile::dateTimeCallback(dateTime);
setSyncProvider(getTeensy3Time);
}
void loop() {
// put your main code here, to run repeatedly:
char ch;
if (Serial.available()) {
ch = Serial.read();
if (ch == 's') WriteAsciiFile(160);
if (ch == 'l') WriteAsciiFile(1000000);
if (ch == 'r') ParseAsciiFile();
if (ch == 'd') sd.ls(LS_SIZE | LS_DATE | LS_R);
}
}
// Write random number to ascii file. Most numbers will be in range 0 to 99, but occasional values
// will be very large. Write one at a time to file, since efficiency isn't an issue in generating the file.
void WriteAsciiFile(uint32_t fnums) {
uint32_t i, num;
Serial.printf("\n\nWriting ASCII File of %lu positive integers\n", fnums);
// Open the file
if (!asciiFile.open(filename, O_RDWR | O_CREAT | O_TRUNC)) {
Serial.printf("Unable to open <%s> for writing.", filename);
return;
}
// start with '<'
asciiFile.print('<');
// Write the numbers as ascii with '.' separator
for (i = 0; i < fnums; i++) {
num = random(99);
if (num == 50) num = random(1311) * 1117;
asciiFile.printf("%u.", num);
if (i <= NUMTODISPLAY) { // show verification output first values
Serial.print(num); Serial.print(".");
if ((i % 16) == 15) Serial.println();
}
}
// end with '>'
asciiFile.print('>');
asciiFile.close();
Serial.println("Write Finished");
}
void ParseAsciiFile(void) {
uint16_t idx, numread, numdigits;
uint32_t totalchars, totalnums, tempnum;
uint32_t startmilli, endmilli;
char ch;
if (!asciiFile.open(filename, O_READ)) {
Serial.printf("\nCould not open <%s> for reading.", filename);
return;
}
startmilli = millis(); // save starting time
Serial.printf("\n\nParsing ASCII File of positive integers\n");
idx = 0;
totalnums = 0;
totalchars = 0;
tempnum = 0;
numdigits = 0;
do {
numread = asciiFile.read(&cbuffer[0], BUFFSIZE);
totalchars += numread;
// numread tells us how many were read. When it is less than BUFFSIZE, we are done
for (idx = 0; idx < numread; idx++) {
ch = cbuffer[idx];
if (isdigit(ch)) { // add to tempnum
tempnum = tempnum * 10 + (ch & 0x0F); // make up the binary number
numdigits++;
} else { // we are at separator, '<' or '>'
if (numdigits > 0) {
if (totalnums <= NUMTODISPLAY) {
SendNumber(tempnum); // send out resulting number
if ((totalnums % 16) == 15) Serial.println();
}
totalnums++;
}
tempnum = 0;
numdigits = 0;
// if there is the possibility of a '>' before the end of the file
// you can add code for early exit here
}
}
} while (numread == BUFFSIZE);
endmilli = millis();
asciiFile.close();
Serial.println();
Serial.printf("Parsing %lu characters into %lu numbers took %lu milliseconds\n",
totalchars, totalnums, endmilli - startmilli);
}
void SendNumber(uint32_t num) {
Serial.printf("%lu ", num);
}
//------------------------------------------------------------------------------
/*
User provided date time callback function.
See SdFile::dateTimeCallback() for usage.
*/
void dateTime(uint16_t* date, uint16_t* time) {
// use the year(), month() day() etc. functions from timelib
// return date using FAT_DATE macro to format fields
*date = FAT_DATE(year(), month(), day());
// return time using FAT_TIME macro to format fields
*time = FAT_TIME(hour(), minute(), second());
}
