Many TLAs: MTP MSC FS SD SDFat LittleFS UsbMSCFat to work with each other 8)

Wiped the whole 128GB SD and added 4 new partitions of diff sizes than before - same result :(
Code:
msc # Partition Table
	part,boot,bgnCHS[3],type,endCHS[3],start,length
FAT32:	1,0,0x20,0x21,0x0,0xC,0xFE,0xFF,0xFF,2048,67108864
exFAT:	2,0,0xFE,0xFF,0xFF,0x7,0xFE,0xFF,0xFF,67110912,134217728
FAT32:	3,0,0xFE,0xFF,0xFF,0xC,0xFE,0xFF,0xFF,201328640,16777216
pt_#15:	4,0,0xFE,0xFF,0xFF,0xF,0xFE,0xFF,0xFF,218105856,26171392
PFsVolume::begin(200047f0)

There is something unique about the 4th. On deleting the disk - the partition then needed to be deleted? Deleting the other three just worked to leave the space open.

Note the 4th when created was tiny 8GB - so the default was FAT32 format - I then reformatted as ExFat?
 
@all, I hacked on the dump MBR to know a little about extended partition...
I have not pushed it up yet...


Code:
//  VolumeName.ino
// VER: P494, p498, p506, p507. p522
//  An example of how to retrieve Fat32 and ExFat volume names using SdFat.
//  Works with SD cards and USB mass storage drives.

#include "Arduino.h"
#include "mscFS.h"

// Setup USBHost_t36 and as many HUB ports as needed.
USBHost myusb;
USBHub hub1(myusb);
USBHub hub2(myusb);
USBHub hub3(myusb);
USBHub hub4(myusb);

#define SHOW_CLOCK_CARAT 1
IntervalTimer clocked100ms;
volatile int32_t cmsReport = -1;

// Setup MSC for the number of USB Drives you are using. (Two for this example)
// Mutiple  USB drives can be used. Hot plugging is supported. There is a slight
// delay after a USB MSC device is plugged in. This is waiting for initialization
// but after it is initialized ther should be no delay.
msController msDrive1(myusb);
msController msDrive2(myusb);

#define SD_DRIVE 1
#define MS_DRIVE 2

#define SD_CONFIG SdioConfig(FIFO_SDIO)

// set up variables using the mscFS utility library functions:
UsbFs msc1;
UsbFs msc2;

SdFs sd;


bool getPartitionVolumeLabel(PFsVolume &partVol, uint8_t *pszVolName, uint16_t cb) {
  uint8_t buf[512];
  if (!pszVolName || (cb < 12)) return false; // don't want to deal with it

  PFsFile root;
  if (!root.openRoot(&partVol)) return false;
  root.read(buf, 32);
  //dump_hexbytes(buf, 32);

  switch (partVol.fatType())
  {
    case FAT_TYPE_FAT12:
    case FAT_TYPE_FAT16:
    case FAT_TYPE_FAT32:
      {
        DirFat_t *dir;
        dir = reinterpret_cast<DirFat_t*>(buf);
        if ((dir->attributes & 0x08) == 0) return false; // not a directory...
        size_t i;
        for (i = 0; i < 11; i++) {
          pszVolName[i]  = dir->name[i];
        }
        while ((i > 0) && (pszVolName[i - 1] == ' ')) i--; // trim off trailing blanks
        pszVolName[i] = 0;
      }
      break;
    case FAT_TYPE_EXFAT:
      {
        DirLabel_t *dir;
        dir = reinterpret_cast<DirLabel_t*>(buf);
        if (dir->type != EXFAT_TYPE_LABEL) return false; // not a label?
        size_t i;
        for (i = 0; i < dir->labelLength; i++) {
          pszVolName[i] = dir->unicode[2 * i];
        }
        pszVolName[i] = 0;
      }
      break;
  }
  return true;
}

typedef struct {
  uint32_t free;
  uint32_t todo;
  uint32_t clusters_per_sector;
} _gfcc_t;


void _getfreeclustercountCB(uint32_t token, uint8_t *buffer) 
{
  digitalWriteFast(1, HIGH);
  _gfcc_t *gfcc = (_gfcc_t *)token;
  uint16_t cnt = gfcc->clusters_per_sector;
  if (cnt > gfcc->todo) cnt = gfcc->todo;
  gfcc->todo -= cnt; // update count here...

  if (gfcc->clusters_per_sector == 512/2) {
    // fat16
    uint16_t *fat16 = (uint16_t *)buffer;
    while (cnt-- ) {
      if (*fat16++ == 0) gfcc->free++;
    }
  } else {
    uint32_t *fat32 = (uint32_t *)buffer;
    while (cnt-- ) {
      if (*fat32++ == 0) gfcc->free++;
    }
  }

  digitalWriteFast(1, LOW);
}

uint32_t GetFreeClusterCount(USBmscInterface *usmsci, PFsVolume &partVol)
{

  FatVolume* fatvol =  partVol.getFatVol();
  if (!fatvol) return 0;

  _gfcc_t gfcc; 
  gfcc.free = 0;

  switch (partVol.fatType()) {
    default: return 0;
    case FAT_TYPE_FAT16: gfcc.clusters_per_sector = 512/2; break;
    case FAT_TYPE_FAT32: gfcc.clusters_per_sector = 512/4; break;
  }
  gfcc.todo = fatvol->clusterCount() + 2;

  digitalWriteFast(0, HIGH);
  usmsci->readSectorsWithCB(fatvol->fatStartSector(), gfcc.todo / gfcc.clusters_per_sector + 1, 
      &_getfreeclustercountCB, (uint32_t)&gfcc);
  digitalWriteFast(0, LOW);

  return gfcc.free;
}

//-------------------------------------------------------------------------------------------------

bool mbrDmpExtended(BlockDeviceInterface *blockDev, uint32_t sector, uint8_t indent) {
  MbrSector_t mbr;
  // bool valid = true;
  if (!blockDev->readSector(sector, (uint8_t*)&mbr)) {
    Serial.print("\nread extended MBR failed.\n");
    //errorPrint();
    return false;
  }
  for (uint8_t ip = 1; ip < 5; ip++) {
    MbrPart_t *pt = &mbr.part[ip - 1];
    //    if ((pt->boot != 0 && pt->boot != 0X80) ||
    //        getLe32(pt->relativeSectors) > sdCardCapacity(&m_csd)) {
    //      valid = false;
    //    }
    for (uint8_t ii=0; ii< indent; ii++)Serial.write('\t');
    switch (pt->type) {
      case 4:
      case 6:
      case 0xe:
        Serial.print("FAT16:\t");
        break;
      case 11:
      case 12:
        Serial.print("FAT32:\t");
        break;
      case 7:
        Serial.print("exFAT:\t");
        break;
      case 0xf:
        Serial.print("Extend:\t");
        break;
      default:
        Serial.print("pt_#");
        Serial.print(pt->type);
        Serial.print(":\t");
        break;
    }
    Serial.print( int(ip)); Serial.print( ',');
    Serial.print(int(pt->boot), HEX); Serial.print( ',');
    for (int i = 0; i < 3; i++ ) {
      Serial.print("0x"); Serial.print(int(pt->beginCHS[i]), HEX); Serial.print( ',');
    }
    Serial.print("0x"); Serial.print(int(pt->type), HEX); Serial.print( ',');
    for (int i = 0; i < 3; i++ ) {
      Serial.print("0x"); Serial.print(int(pt->endCHS[i]), HEX); Serial.print( ',');
    }
    uint32_t starting_sector = getLe32(pt->relativeSectors);
    Serial.print(starting_sector, DEC); Serial.print(',');
    Serial.println(getLe32(pt->totalSectors));

    // for fun of it... try printing extended data
    if (pt->type == 0xf) mbrDmpExtended(blockDev, starting_sector, indent+1);
  }
  return true;
}
//-------------------------------------------------------------------------------------------------

bool mbrDmp(BlockDeviceInterface *blockDev) {
  MbrSector_t mbr;
  // bool valid = true;
  if (!blockDev->readSector(0, (uint8_t*)&mbr)) {
    Serial.print("\nread MBR failed.\n");
    //errorPrint();
    return false;
  }
  Serial.print("\nmsc # Partition Table\n");
  Serial.print("\tpart,boot,bgnCHS[3],type,endCHS[3],start,length\n");
  for (uint8_t ip = 1; ip < 5; ip++) {
    MbrPart_t *pt = &mbr.part[ip - 1];
    //    if ((pt->boot != 0 && pt->boot != 0X80) ||
    //        getLe32(pt->relativeSectors) > sdCardCapacity(&m_csd)) {
    //      valid = false;
    //    }
    switch (pt->type) {
      case 4:
      case 6:
      case 0xe:
        Serial.print("FAT16:\t");
        break;
      case 11:
      case 12:
        Serial.print("FAT32:\t");
        break;
      case 7:
        Serial.print("exFAT:\t");
        break;
      case 0xf:
        Serial.print("Extend:\t");
        break;
      default:
        Serial.print("pt_#");
        Serial.print(pt->type);
        Serial.print(":\t");
        break;
    }
    Serial.print( int(ip)); Serial.print( ',');
    Serial.print(int(pt->boot), HEX); Serial.print( ',');
    for (int i = 0; i < 3; i++ ) {
      Serial.print("0x"); Serial.print(int(pt->beginCHS[i]), HEX); Serial.print( ',');
    }
    Serial.print("0x"); Serial.print(int(pt->type), HEX); Serial.print( ',');
    for (int i = 0; i < 3; i++ ) {
      Serial.print("0x"); Serial.print(int(pt->endCHS[i]), HEX); Serial.print( ',');
    }
    uint32_t starting_sector = getLe32(pt->relativeSectors);
    Serial.print(starting_sector, DEC); Serial.print(',');
    Serial.println(getLe32(pt->totalSectors));

    // for fun of it... try printing extended data
    if (pt->type == 0xf) mbrDmpExtended(blockDev, starting_sector, 1);
  }
  return true;
}

void setup()
{
#if 0 // easy test to check HardFault Detection response
  int *pp = 0;
  *pp = 5;
#endif
pinMode(0, OUTPUT);
pinMode(1, OUTPUT);
pinMode(2, OUTPUT);
  // Open serial communications and wait for port to open:
  Serial.begin(9600);
  while (!Serial) {
    SysCall::yield(); // wait for serial port to connect.
  }

  // Start USBHost_t36, HUB(s) and USB devices.
  myusb.begin();

#ifdef SHOW_CLOCK_CARAT
  clocked100ms.begin(clock_isr, 100000);
#endif

}

void clock_isr() {
  if (cmsReport >= 0 ) {
    if (cmsReport > 0 ) {
      if (cmsReport < 10 )
        Serial.print( "^");
      else if ( cmsReport < 50 && !(cmsReport % 10) )
        Serial.print( "~");
      else if ( !(cmsReport % 50) )
        Serial.print( ":(");
    }
    cmsReport++;
  }
}

void dump_hexbytes(const void *ptr, int len)
{
  if (ptr == NULL || len <= 0) return;
  const uint8_t *p = (const uint8_t *)ptr;
  while (len) {
    for (uint8_t i = 0; i < 32; i++) {
      if (i > len) break;
      Serial.printf("%02X ", p[i]);
    }
    Serial.print(":");
    for (uint8_t i = 0; i < 32; i++) {
      if (i > len) break;
      Serial.printf("%c", ((p[i] >= ' ') && (p[i] <= '~')) ? p[i] : '.');
    }
    Serial.println();
    p += 32;
    len -= 32;
  }
}



// Function to handle one MS Drive...
void procesMSDrive(uint8_t drive_number, msController &msDrive, UsbFs &msc)
{
  Serial.printf("Initialize USB drive...");
  cmsReport = 0;
  if (!msc.begin(&msDrive)) {
    Serial.println("");
    msc.errorPrint(&Serial);
    Serial.printf("initialization drive %u failed.\n", drive_number);
  } else {
    Serial.printf("USB drive %u is present.\n", drive_number);
  }
  cmsReport = -1;

  //  mbrDmp( &msc );
  mbrDmp( msc.usbDrive() );

  #if 1
  bool partition_valid[4];
  PFsVolume partVol[4];
  uint8_t volName[32];

  for (uint8_t i = 0; i < 4; i++) {
    partition_valid[i] = partVol[i].begin(msc.usbDrive(), true, i+1);
    Serial.printf("Partition %u valid:%u\n", i, partition_valid[i]);
  }
  for (uint8_t i = 0; i < 4; i++) {
    if(partition_valid[i]) {
      switch (partVol[i].fatType())
      {
        case FAT_TYPE_FAT12: Serial.printf("%d:>> Fat12: ", i); break;
        case FAT_TYPE_FAT16: Serial.printf("%d:>> Fat16: ", i); break;
        case FAT_TYPE_FAT32: Serial.printf("%d:>> Fat32: ", i); break;
        case FAT_TYPE_EXFAT: Serial.printf("%d:>> ExFat: ", i); break;
      }
      if (getPartitionVolumeLabel(partVol[i], volName, sizeof(volName))) {
        Serial.printf("Volume name:(%s)", volName);
      }
      elapsedMicros em_sizes = 0;
      uint32_t free_cluster_count = partVol[i].freeClusterCount();
      uint64_t used_size =  (uint64_t)(partVol[i].clusterCount() - free_cluster_count)
                            * (uint64_t)partVol[i].bytesPerCluster();
      uint64_t total_size = (uint64_t)partVol[i].clusterCount() * (uint64_t)partVol[i].bytesPerCluster();
      Serial.printf(" Partition Total Size:%llu Used:%llu time us: %u\n", total_size, used_size, (uint32_t)em_sizes);

      em_sizes = 0; // lets see how long this one takes. 
      uint32_t free_clusters_fast = GetFreeClusterCount(msc.usbDrive(), partVol[i]);
      Serial.printf("    Free Clusters: API: %u by CB:%u time us: %u\n", free_cluster_count, free_clusters_fast, (uint32_t)em_sizes);
      partVol[i].ls();
    }
  }

  #else
  for (uint8_t i = 1; i < 5; i++) {
    PFsVolume partVol;
    uint8_t volName[32];
    if (!partVol.begin(msc.usbDrive(), true, i)) continue; // not a valid volume.
    partVol.chvol();

    switch (partVol.fatType())
    {
      case FAT_TYPE_FAT12: Serial.print("\n>> Fat12: "); break;
      case FAT_TYPE_FAT16: Serial.print("\n>> Fat16: "); break;
      case FAT_TYPE_FAT32: Serial.print("\n>> Fat32: "); break;
      case FAT_TYPE_EXFAT: Serial.print("\n>> ExFat: "); break;
    }
    if (getPartitionVolumeLabel(partVol, volName, sizeof(volName))) {
      Serial.printf("Volume name:(%s)", volName);
    }
    elapsedMicros em_sizes = 0;
    uint64_t used_size =  (uint64_t)(partVol.clusterCount() - partVol.freeClusterCount())
                          * (uint64_t)partVol.bytesPerCluster();
    uint64_t total_size = (uint64_t)partVol.clusterCount() * (uint64_t)partVol.bytesPerCluster();
    Serial.printf(" Partition Total Size:%llu Used:%llu time us: %u\n", total_size, used_size, (uint32_t)em_sizes);

    partVol.ls();
  }
  #endif
}

//================================================================
void loop(void) {
  //--------------------------------------------------
  cmsReport = 0;
  myusb.Task();
  if (!msDrive1) {
    Serial.println("Waiting up to 5 seconds for USB drive 1");
    elapsedMillis em = 0;
    while (!msDrive1 && (em < 5000) )  myusb.Task();
  }
  if (!msDrive2) {
    Serial.println("Waiting up to 5 seconds for USB drive 2");
    elapsedMillis em = 0;
    while (!msDrive2 && (em < 5000) )  myusb.Task();
  }
  if (msDrive1) {
    procesMSDrive(1, msDrive1, msc1);
  }
  if (msDrive2) {
    procesMSDrive(2, msDrive2, msc2);
  }
  cmsReport = -1;
  //--------------------------------------------------
  Serial.printf("\nInitialize SD card...");

  if (!sd.begin(SD_CONFIG)) {
    Serial.println("initialization failed.\n");
  } else {
    Serial.println("SD card is present.\n");
    mbrDmp(sd.card() );
    PFsVolume partVol;

    for (uint8_t i = 1; i < 5; i++) {
      PFsVolume partVol;
      uint8_t volName[32];
      if (!partVol.begin(sd.card(), true, i)) continue; // not a valid volume.
      partVol.chvol();

      switch (partVol.fatType())
      {
        case FAT_TYPE_FAT12: Serial.print("\n>> Fat12: "); break;
        case FAT_TYPE_FAT16: Serial.print("\n>> Fat16: "); break;
        case FAT_TYPE_FAT32: Serial.print("\n>> Fat32: "); break;
        case FAT_TYPE_EXFAT: Serial.print("\n>> ExFat: "); break;
      }
      if (getPartitionVolumeLabel(partVol, volName, sizeof(volName))) {
        Serial.printf("Volume name:(%s)", volName);
      }
      elapsedMicros em_sizes = 0;
      uint64_t used_size =  (uint64_t)(partVol.clusterCount() - partVol.freeClusterCount())
                            * (uint64_t)partVol.bytesPerCluster();
      uint64_t total_size = (uint64_t)partVol.clusterCount() * (uint64_t)partVol.bytesPerCluster();
      Serial.printf(" Partition Total Size:%llu Used:%llu time us: %u\n", total_size, used_size, (uint32_t)em_sizes);

      partVol.ls();
    }
  }

  Serial.println("done...");

  Serial.println("Press any key to run again");
  while (Serial.read() == -1);
  while (Serial.read() != -1);
}

Again SDFat code knows nothing on how to handle these, but a test disk to have an extended...
Code:
msc # Partition Table
	part,boot,bgnCHS[3],type,endCHS[3],start,length
FAT32:	1,0,0x4,0x1,0x4,0xB,0xFE,0xC2,0xFF,2048,4096000
exFAT:	2,0,0x17,0x19,0xFF,0x7,0xE,0x50,0xFE,4098048,4096000
FAT32:	3,0,0xE,0x51,0xFE,0xC,0x5,0x88,0xFD,8194048,4096000
Extend:	4,0,0x5,0x89,0xFD,0xF,0xFE,0xFF,0xD2,12290048,3438592
	FAT32:	1,0,0x20,0x21,0x0,0xC,0x1C,0x36,0x66,2048,1638400
	pt_#5:	2,0,0x1C,0x37,0x66,0x5,0xA,0x34,0xD6,1640448,1798144
	pt_#0:	3,0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0,0
	pt_#0:	4,0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0,0
PFsVolume::begin(20002070)
Probably the interesting thing to make anything work with it, is my guess is that the Sectors/clusters in the extended partitions are relative to the start of the extended...
 
The question is, what is the 4th partition? It shows type: 0xf
Looking at: https://en.wikipedia.org/wiki/Partition_type
It is an Extended Partition indicator! So
the data for your actual partition would probably be in the logical record: 218105856

I didn't check the numbers for the pt_#15 [glad I added that to that code :)] - but on deleting and find it saying 'delete partition' my guess was it created an extended part to hold the 4th partition.

I've just lost hours doing copyPasta to write collected numbers to SD ... somehow using if (myFile) when it should have been if (!myFile) ... so ignoring that ... I spent hours searching/replacing all the code ABOVE that wondering why I was getting "open failed" ... :(

Now to read the data I wrote in binary for smaller uniform writes - that still adds a lot of time writing them all ... but spewing even 1 in 100 of them over USB was being a headache ... but that's another distraction.
 
The question is, what is the 4th partition? It shows type: 0xf
Looking at: https://en.wikipedia.org/wiki/Partition_type
It is an Extended Partition indicator! So
the data for your actual partition would probably be in the logical record: 218105856

Yes I just woke up for a little while.

Anyway that makes sense. I looked at the my 4th partition before I reformatted. Windows showed it as a "Logical" partition. All the other partitions were identified as "Primary". So when I reformatted it in some other partition software I told it format as a FAT16 Primary partition from the FAT16 Logical partition.
 
@kurtE - just downloaded your changes and after running some errands I can't wait to try them out. I noticed that you changed the print and println's back to Serial.printf's again. @Paul had asked me to change these to print and println before the next beta release for debug reasons. I guess I am a little confused.

I just pushed up a updated version of it that has a lot less changes to the print stuff.
I did a git rebase -i ...
Update as to not have it show that I changed them all back to old and then all back to new...

Note: I was/am tempted to still change some of this debug output:
That is you for example have:
Code:
uint8_t msController::mscInit(void) {
#ifdef DBGprint
	println("mscIint()");
#endif
Now the library is setup that println will not output anything unless line:
//#define USBHOST_PRINT_DEBUG
is uncommented in USBHost_t36.h file.

In the current file you had: #define DBGprint 0
I left it as commented out value 1 as you had before not sure of why the changes.

Maybe you meant to change all of these #ifdef DBGprint to #if DBGPrint > 0
Or some such thing?

Again note: in some of the other classes we have additional debug stuff, that allows a couple of different levels of debug messages:
Like in Bluetooth we have:
Code:
#define print   USBHost::print_
#define println USBHost::println_//#define DEBUG_BT

#define DEBUG_BT
//#define DEBUG_BT_VERBOSE

#ifndef DEBUG_BT
#undef DEBUG_BT_VERBOSE
void inline DBGPrintf(...) {};
#else
#define DBGPrintf USBHDBGSerial.printf
#endif

#ifndef DEBUG_BT_VERBOSE
void inline VDBGPrintf(...) {};
#else
#define VDBGPrintf USBHDBGSerial.printf
#endif
Which comes in handy. In this case we did not hard code in the file the the use of Serial as the debug output... It uses what again we define in USBHost_t36.h file, so we may decide
for example to user SerialUSB1 or Serial4 or ....

Also got rid of most of the #if stuff in the code except when the debug code needed additional steps to be done. So for example instead of using Serial.printf(...)
I used DBGPrintf(...);
So when the debug is not set this is setup to be inline function that does nothing... So should simply disappear...
 
Assuming I downloaded the correct branches looks like the changes are working. Not going to add the dumps but here is just the results for no VolumeName with debug prints turned off in USBHost_t36:
Code:
Waiting up to 5 seconds for USB drive 1
Waiting up to 5 seconds for USB drive 2
^^^^^^^^^~~~~:(Initialize USB drive...UsbBase::mscBegin called 200029c0 1 1
^^^    After usbDriveBegin
PFsVolume::begin(20002510)
USB drive 1 is present.

msc # Partition Table
	part,boot,bgnCHS[3],type,endCHS[3],start,length
FAT32:	1,80,0x20,0x21,0x0,0xC,0xC2,0x22,0x20,2048,524288
FAT32:	2,0,0xC2,0x23,0x20,0xC,0xFE,0xFF,0xFF,526336,16564224
exFAT:	3,0,0xFE,0xFF,0xFF,0x7,0xFE,0xFF,0xFF,17090560,228452352
FAT16:	4,0,0xFE,0xFF,0xFF,0xE,0xFE,0xFF,0xFF,245542912,4194304
PFsVolume::begin(20002510)
Partition 0 valid:1
PFsVolume::begin(20002510)
Partition 1 valid:1
PFsVolume::begin(20002510)
Partition 2 valid:1
PFsVolume::begin(20002510)
Partition 3 valid:1
0:>> Fat32: Volume name:(system-boot)###FatPartition::freeClusterCount: FT:32 start:2080 todo:516192
    m_sectorsPerCluster:1
    m_clusterSectorMask:0
    m_sectorsPerClusterShift:0
    m_fatType:32
    m_rootDirEntryCount:0
    m_allocSearchStart:1
    m_sectorsPerFat:4033
    m_dataStartSector:10146
    m_fatStartSector:2080
    m_lastCluster:516191
    m_rootDirStart:2
    m_bytesPerSector:512
 Partition Total Size:264289280 Used:961024 time us: 2131123
    Free Clusters: API: 514313 by CB:514313 time us: 252491
bench0.txt
bench1.txt
bench2.txt
bench3.txt
bench4.txt
bench5.txt
1:>> Fat32: Volume name:(NEW VOLUME1)###FatPartition::freeClusterCount: FT:32 start:526814 todo:2066434
    m_sectorsPerCluster:8
    m_clusterSectorMask:7
    m_sectorsPerClusterShift:3
    m_fatType:32
    m_rootDirEntryCount:0
    m_allocSearchStart:1
    m_sectorsPerFat:16145
    m_dataStartSector:559104
    m_fatStartSector:526814
    m_lastCluster:2066433
    m_rootDirStart:2
    m_bytesPerSector:512
 Partition Total Size:8464105472 Used:983040 time us: 8433248
    Free Clusters: API: 2066192 by CB:2066192 time us: 1009491
bench0.txt
bench1.txt
bench2.txt
bench3.txt
bench4.txt
bench5.txt
2:>> ExFat: Volume name:(exFAT-vol3) Partition Total Size:116962361344 Used:786432 time us: 109002
    Free Clusters: API: 892346 by CB:0 time us: 0
3:>> Fat16: Volume name:(FAT16)###FatPartition::freeClusterCount: FT:16 start:245542913 todo:32767
    m_sectorsPerCluster:128
    m_clusterSectorMask:127
    m_sectorsPerClusterShift:7
    m_fatType:16
    m_rootDirEntryCount:512
    m_allocSearchStart:1
    m_sectorsPerFat:128
    m_dataStartSector:245543201
    m_fatStartSector:245542913
    m_lastCluster:32766
    m_rootDirStart:245543169
    m_bytesPerSector:512
 Partition Total Size:0 Used:0 time us: 64000
    Free Clusters: API: 32762 by CB:32762 time us: 8369

Initialize SD card...initialization failed.

done...
Press any key to run again
 
Yes, in my local branches I commented out some of the printing... FatPartition.cpp
in SDFat code...
Code:
int32_t FatPartition::freeClusterCount() {
#if MAINTAIN_FREE_CLUSTER_COUNT
  if (m_freeClusterCount >= 0) {
    return m_freeClusterCount;
  }
#endif  // MAINTAIN_FREE_CLUSTER_COUNT
  uint32_t free = 0;
  uint32_t sector;
  uint32_t todo = m_lastCluster + 1;
  uint16_t n;

  if (FAT12_SUPPORT && fatType() == 12) {
    for (unsigned i = 2; i < todo; i++) {
      uint32_t c;
      int8_t fg = fatGet(i, &c);
      if (fg < 0) {
        DBG_FAIL_MACRO;
        goto fail;
      }
      if (fg && c == 0) {
        free++;
      }
    }
  } else if (fatType() == 16 || fatType() == 32) {
#if 0    
    Serial.printf("###FatPartition::freeClusterCount: FT:%u start:%u todo:%u\n", fatType(), m_fatStartSector, todo);
    Serial.printf("    m_sectorsPerCluster:%u\n", m_sectorsPerCluster);
    Serial.printf("    m_clusterSectorMask:%u\n", m_clusterSectorMask);
    Serial.printf("    m_sectorsPerClusterShift:%u\n", m_sectorsPerClusterShift);
    Serial.printf("    m_fatType:%u\n", m_fatType);
    Serial.printf("    m_rootDirEntryCount:%u\n", m_rootDirEntryCount);
    Serial.printf("    m_allocSearchStart:%u\n", m_allocSearchStart);
    Serial.printf("    m_sectorsPerFat:%u\n", m_sectorsPerFat);
    Serial.printf("    m_dataStartSector:%u\n", m_dataStartSector);
    Serial.printf("    m_fatStartSector:%u\n", m_fatStartSector);
    Serial.printf("    m_lastCluster:%u\n", m_lastCluster);
    Serial.printf("    m_rootDirStart:%u\n", m_rootDirStart);
    Serial.printf("    m_bytesPerSector:%u\n", m_bytesPerSector);
#endif
I added the prints earlier to figure out why it was reading so much data... Turned off in my current runs.
 
@KurtE - I finally found the information about FAT16/FAT32 FS Information Sector here:

https://en.wikipedia.org/wiki/Design_of_the_FAT_file_system.

I am not sure if you knew about it. Unfortunately SdFat does not process or use it. FatFS does. Using this avoids having to scan for free cluster unless the count is invalid. It is updated in several parts of the FatFS code.

Thanks, I had not looked at the info sector as, you mentioned it is not used in SDFat... Not sure how hard it would be to use it.

There is optional code in Fat code that if enabled when you ask for the count of free sectors, if first time, it will do the scan, and then remember it. Then in places that allocate clusters it will update the count...

My guess is we probably don't want to take on the info sector yet...
 
Thanks, I had not looked at the info sector as, you mentioned it is not used in SDFat... Not sure how hard it would be to use it.

There is optional code in Fat code that if enabled when you ask for the count of free sectors, if first time, it will do the scan, and then remember it. Then in places that allocate clusters it will update the count...

My guess is we probably don't want to take on the info sector yet...

It is certainly above my pay grade:)

There is optional code in Fat code that if enabled when you ask for the count of free sectors, if first time, it will do the scan, and then remember it. Then in places that allocate clusters it will update the count...

That would seem to be the easiest. And is very close to using 'uint8_t fat32FSInfoSector[2];'.
Now to do some more testing with your latest updates.
 
Yes, in my local branches I commented out some of the printing... FatPartition.cpp
in SDFat code...
....
I added the prints earlier to figure out why it was reading so much data... Turned off in my current runs.

I downloaded the _cb branch - hope thats the correct branch
 
Morning all,

The info sector is interesting thing, may add another hack onto the VolumeName sketch version which tries to read it. I am sort of curious for example if I then transfer a stick to Windows, does it detect that the data is not valid there and maybe update it? Does it make sense for our code to try to use it? Will be interesting.
 
Morning all,

The info sector is interesting thing, may add another hack onto the VolumeName sketch version which tries to read it. I am sort of curious for example if I then transfer a stick to Windows, does it detect that the data is not valid there and maybe update it? Does it make sense for our code to try to use it? Will be interesting.

Good morning all as well

Sounds fun. I started playing with trying to get a kludged together fatformater working, going piece by piece but knowledge getting stuck on c++ issue. I started with just the basic call to format function:
Code:
#include "PFsFatFormatter.h"
// Set nonzero to use calculated CHS in MBR.  Should not be required.
#define USE_LBA_TO_CHS 1

// Constants for file system structure optimized for flash.
uint16_t const BU16 = 128;
uint16_t const BU32 = 8192;
// Assume 512 byte sectors.
const uint16_t BYTES_PER_SECTOR = 512;
const uint16_t SECTORS_PER_MB = 0X100000/BYTES_PER_SECTOR;
const uint16_t FAT16_ROOT_ENTRY_COUNT = 512;
const uint16_t FAT16_ROOT_SECTOR_COUNT =
               32*FAT16_ROOT_ENTRY_COUNT/BYTES_PER_SECTOR;
//------------------------------------------------------------------------------
#define PRINT_FORMAT_PROGRESS 1
#if !PRINT_FORMAT_PROGRESS
#define writeMsg(str)
#elif defined(__AVR__)
#define writeMsg(str) if (m_pr) m_pr->print(F(str))
#else  // PRINT_FORMAT_PROGRESS
#define writeMsg(str) if (m_pr) m_pr->write(str)
#endif  // PRINT_FORMAT_PROGRESS
//------------------------------------------------------------------------------
bool PFsFatFormatter::format(PFsVolume &partVol, uint8_t* secBuf, print_t* pr, uint8_t part) {
  bool rtn;

  m_secBuf = secBuf;
  m_pr = pr;
  m_sectorCount = partVol->sectorCount();
  m_capacityMB = (m_sectorCount + SECTORS_PER_MB - 1)/SECTORS_PER_MB;
  
  Serial.printf("Partition Capacity: %d\n", m_capacityMB);

  if (m_capacityMB <= 6) {
    writeMsg("Card is too small.\r\n");
    return false;
  } else if (m_capacityMB <= 16) {
    m_sectorsPerCluster = 2;
  } else if (m_capacityMB <= 32) {
    m_sectorsPerCluster = 4;
  } else if (m_capacityMB <= 64) {
    m_sectorsPerCluster = 8;
  } else if (m_capacityMB <= 128) {
    m_sectorsPerCluster = 16;
  } else if (m_capacityMB <= 1024) {
    m_sectorsPerCluster = 32;
  } else if (m_capacityMB <= 32768) {
    m_sectorsPerCluster = 64;
  } else {
    // SDXC cards
    m_sectorsPerCluster = 128;
  }
[COLOR="#B22222"]  //rtn = m_sectorCount < 0X400000 ? makeFat16() :makeFat32();
[/COLOR]  if (rtn) {
    writeMsg("Format Done\r\n");
  } else {
    writeMsg("Format Failed\r\n");
  }
  return rtn;
}
Notice I commented out the essential call to actually format the volume. Anyway getting this error:
Code:
error: 'PFsVolume' has not been declared
Want to make my life easier - how can I fix?
 
@mjs513 - Good morning.

Probably need to include: UsbMscFat\.....\PFsLib\PFsVolume.h
Which is probably included in some other higher level include files.
Like:
#include "mscFS.h"
 
OK I have a quick and dirty version of the code to read the information sector for free space...
Code:
uint32_t ReadFat32InfoSectorFree(USBmscInterface *usmsci, uint8_t part) {
  uint8_t sector_buffer[512];

  
  if (!usmsci->readSector(0, sector_buffer)) return (uint32_t)-1;
  MbrSector_t *mbr = reinterpret_cast<MbrSector_t*>(sector_buffer);
  MbrPart_t *pt = &mbr->part[part - 1];
  BpbFat32_t* bpb;
  if ((pt->type != 11) && (pt->type != 12))  return (uint32_t)-1;

  uint32_t volumeStartSector = getLe32(pt->relativeSectors);
  if (!usmsci->readSector(volumeStartSector, sector_buffer)) return (uint32_t)-1;
  pbs_t *pbs = reinterpret_cast<pbs_t*> (sector_buffer);
  bpb = reinterpret_cast<BpbFat32_t*>(pbs->bpb);
  
  Serial.println("\nReadFat32InfoSectorFree BpbFat32_t sector");
  dump_hexbytes(sector_buffer, 512);
  uint16_t infoSector = getLe16(bpb->fat32FSInfoSector); 

  // I am assuming this sector is based off of the volumeStartSector... So try reading from there.
  Serial.printf("Try to read Info sector (%u)\n", infoSector); Serial.flush(); 
  if (!usmsci->readSector(volumeStartSector+infoSector, sector_buffer)) return (uint32_t)-1;
  dump_hexbytes(sector_buffer, 512);
  FsInfo_t *pfsi = reinterpret_cast<FsInfo_t*>(sector_buffer);

  // check signatures:
  if (memcmp(pfsi->leadSignature, "RRaA", 4) != 0) Serial.println("Lead Sig wrong");    
  if (memcmp(pfsi->structSignature, "rrAa", 4) != 0) Serial.println("struct Sig wrong");    
  static const uint8_t _trail_sig[4] = {0x00, 0x00, 0x55, 0xAA};
  if (memcmp(pfsi->trailSignature, _trail_sig, 4) != 0) Serial.println("Trail Sig wrong");    
  uint32_t free_count = getLe32(pfsi->freeCount);
  return free_count;

}

In the ProcessMSDrive I added calls off to it right before the call to use ls...
Code:
      em_sizes = 0; // lets see how long this one takes. 
      uint32_t free_clusters_fast = GetFreeClusterCount(msc.usbDrive(), partVol[i]);
      Serial.printf("    Free Clusters: API: %u by CB:%u time us: %u\n", free_cluster_count, free_clusters_fast, (uint32_t)em_sizes);
      
      em_sizes = 0; // lets see how long this one takes. 
      uint32_t free_clusters_info = ReadFat32InfoSectorFree(msc.usbDrive(), i+1);
      Serial.printf("    Free Clusters: Info: %u time us: %u\n", free_clusters_info, (uint32_t)em_sizes);


Note on my memory stick I tested it on, I think the only files I wrote to it were on Windows so the counts match: Here is a debug run...
Code:
Waiting up to 5 seconds for USB drive 1
^^Waiting up to 5 seconds for USB drive 2
^^^^^^^~~~~:(Initialize USB drive...UsbBase::mscBegin called 20002540 1 1
^^^^^^    After usbDriveBegin
PFsVolume::begin(20002090)
ExFatVolume::begin(20002090 1 1
USB drive 1 is present.

msc # Partition Table
	part,boot,bgnCHS[3],type,endCHS[3],start,length
FAT32:	1,0,0x4,0x1,0x4,0xB,0xFE,0xC2,0xFF,2048,4096000
exFAT:	2,0,0x17,0x19,0xFF,0x7,0xE,0x50,0xFE,4098048,4096000
FAT32:	3,0,0xE,0x51,0xFE,0xC,0x5,0x88,0xFD,8194048,4096000
Extend:	4,0,0x5,0x89,0xFD,0xF,0xFE,0xFF,0xD2,12290048,3438592
	FAT32:	1,0,0x20,0x21,0x0,0xC,0x1C,0x36,0x66,2048,1638400
	pt_#5:	2,0,0x1C,0x37,0x66,0x5,0xA,0x34,0xD6,1640448,1798144
	pt_#0:	3,0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0,0
	pt_#0:	4,0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0,0
PFsVolume::begin(20002090)
ExFatVolume::begin(20002090 1 1
Partition 0 valid:1
PFsVolume::begin(20002090)
ExFatVolume::begin(20002090 1 2
   After init
   After chdir
Partition 1 valid:1
PFsVolume::begin(20002090)
ExFatVolume::begin(20002090 1 3
Partition 2 valid:1
PFsVolume::begin(20002090)
ExFatVolume::begin(20002090 1 4
Partition 3 valid:0
0:>> Fat32: Volume name:(FAT32) Partition Total Size:2093039616 Used:20480 time us: 3992748
    Free Clusters: API: 510991 by CB:510991 time us: 250491

ReadFat32InfoSectorFree BpbFat32_t sector
EB 58 90 6D 6B 66 73 2E 66 61 74 00 02 08 20 00 02 00 00 00 00 F8 00 00 3E 00 F8 00 00 08 00 00 :.X.mkfs.fat... .........>.......
00 80 3E 00 A0 0F 00 00 00 00 00 00 02 00 00 00 01 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 :..>.............................
80 00 29 EF C9 6B 85 46 41 54 33 32 20 20 20 20 20 20 46 41 54 33 32 20 20 20 0E 1F BE 77 7C AC :..)..k.FAT32      FAT32   ...w|.
22 C0 74 0B 56 B4 0E BB 07 00 CD 10 5E EB F0 32 E4 CD 16 CD 19 EB FE 54 68 69 73 20 69 73 20 6E :".t.V.......^..2.......This is n
6F 74 20 61 20 62 6F 6F 74 61 62 6C 65 20 64 69 73 6B 2E 20 20 50 6C 65 61 73 65 20 69 6E 73 65 :ot a bootable disk.  Please inse
72 74 20 61 20 62 6F 6F 74 61 62 6C 65 20 66 6C 6F 70 70 79 20 61 6E 64 0D 0A 70 72 65 73 73 20 :rt a bootable floppy and..press 
61 6E 79 20 6B 65 79 20 74 6F 20 74 72 79 20 61 67 61 69 6E 20 2E 2E 2E 20 0D 0A 00 00 00 00 00 :any key to try again ... .......
00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 :................................
00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 :................................
00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 :................................
00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 :................................
00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 :................................
00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 :................................
00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 :................................
00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 :................................
00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 55 AA :..............................U.
Try to read Info sector (1)
52 52 61 41 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 :RRaA............................
00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 :................................
00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 :................................
00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 :................................
00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 :................................
00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 :................................
00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 :................................
00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 :................................
00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 :................................
00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 :................................
00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 :................................
00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 :................................
00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 :................................
00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 :................................
00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 :................................
00 00 00 00 72 72 41 61 0F CC 07 00 07 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 55 AA :....rrAa......................U.
    Free Clusters: Info: 510991 time us: 5420
OnFat32.txt
Hummingbirds.jpg
1:>> ExFat: Volume name:(ExFat) Partition Total Size:2096824320 Used:3571712 time us: 16001
    Free Clusters: API: 63881 by CB:0 time us: 0
    Free Clusters: Info: 4294967295 time us: 1609
OnExFat.txt
DSC03357.JPG
2:>> Fat32: Volume name:(FAT32) Partition Total Size:2092957696 Used:3391488 time us: 3992748
    Free Clusters: API: 510148 by CB:510148 time us: 250491

ReadFat32InfoSectorFree BpbFat32_t sector
EB 58 90 4D 53 44 4F 53 35 2E 30 00 02 08 CE 00 02 00 00 00 00 F8 00 00 3F 00 FF 00 00 08 7D 00 :.X.MSDOS5.0.............?.....}.
00 80 3E 00 99 0F 00 00 00 00 00 00 02 00 00 00 01 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 :..>.............................
80 00 29 DF AB 22 5E 4E 4F 20 4E 41 4D 45 20 20 20 20 46 41 54 33 32 20 20 20 33 C9 8E D1 BC F4 :..).."^NO NAME    FAT32   3.....
7B 8E C1 8E D9 BD 00 7C 88 56 40 88 4E 02 8A 56 40 B4 41 BB AA 55 CD 13 72 10 81 FB 55 AA 75 0A :{......|.V@.N..V@.A..U..r...U.u.
F6 C1 01 74 05 FE 46 02 EB 2D 8A 56 40 B4 08 CD 13 73 05 B9 FF FF 8A F1 66 0F B6 C6 40 66 0F B6 :...t..F..-.V@....s......f...@f..
D1 80 E2 3F F7 E2 86 CD C0 ED 06 41 66 0F B7 C9 66 F7 E1 66 89 46 F8 83 7E 16 00 75 39 83 7E 2A :...?.......Af...f..f.F..~..u9.~*
00 77 33 66 8B 46 1C 66 83 C0 0C BB 00 80 B9 01 00 E8 2C 00 E9 A8 03 A1 F8 7D 80 C4 7C 8B F0 AC :.w3f.F.f..........,......}..|...
84 C0 74 17 3C FF 74 09 B4 0E BB 07 00 CD 10 EB EE A1 FA 7D EB E4 A1 7D 80 EB DF 98 CD 16 CD 19 :..t.<.t............}...}........
66 60 80 7E 02 00 0F 84 20 00 66 6A 00 66 50 06 53 66 68 10 00 01 00 B4 42 8A 56 40 8B F4 CD 13 :f`.~.... .fj.fP.Sfh.....B.V@....
66 58 66 58 66 58 66 58 EB 33 66 3B 46 F8 72 03 F9 EB 2A 66 33 D2 66 0F B7 4E 18 66 F7 F1 FE C2 :fXfXfXfX.3f;F.r...*f3.f..N.f....
8A CA 66 8B D0 66 C1 EA 10 F7 76 1A 86 D6 8A 56 40 8A E8 C0 E4 06 0A CC B8 01 02 CD 13 66 61 0F :..f..f....v....V@............fa.
82 74 FF 81 C3 00 02 66 40 49 75 94 C3 42 4F 4F 54 4D 47 52 20 20 20 20 00 00 00 00 00 00 00 00 :.t.....f@Iu..BOOTMGR    ........
00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 :................................
00 00 00 00 00 00 00 00 00 00 00 00 0D 0A 44 69 73 6B 20 65 72 72 6F 72 FF 0D 0A 50 72 65 73 73 :..............Disk error...Press
20 61 6E 79 20 6B 65 79 20 74 6F 20 72 65 73 74 61 72 74 0D 0A 00 00 00 00 00 00 00 00 00 00 00 : any key to restart.............
00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 AC 01 B9 01 00 00 55 AA :..............................U.
Try to read Info sector (1)
52 52 61 41 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 :RRaA............................
00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 :................................
00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 :................................
00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 :................................
00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 :................................
00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 :................................
00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 :................................
00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 :................................
00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 :................................
00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 :................................
00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 :................................
00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 :................................
00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 :................................
00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 :................................
00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 :................................
00 00 00 00 72 72 41 61 C4 C8 07 00 3E 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 55 AA :....rrAa....>.................U.
    Free Clusters: Info: 510148 time us: 7295
OnFat32.txt
DSC03355.JPG

Initialize SD card...FsVolume::begin(20004710)
ExFatVolume::begin(20004710 0 1
   After init
   After chdir
SD card is present.


msc # Partition Table
	part,boot,bgnCHS[3],type,endCHS[3],start,length
exFAT:	1,0,0x82,0x3,0x0,0x7,0xFE,0xFF,0xFF,8192,62543872
pt_#0:	2,0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0,0
pt_#0:	3,0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0,0
pt_#0:	4,0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0,0
PFsVolume::begin(20004710)
ExFatVolume::begin(20004710 1 1
   After init
   After chdir

>> ExFat: Volume name:(EXFat_Vol) Partition Total Size:32017219584 Used:4915200 time us: 7270
T4.1-Cardlike.jpg
T4-Cardlike.jpg
DSC03355.JPG
test1.txt
mtpindex.dat
PFsVolume::begin(20004710)
ExFatVolume::begin(20004710 1 2
PFsVolume::begin(20004710)
ExFatVolume::begin(20004710 1 3
PFsVolume::begin(20004710)
ExFatVolume::begin(20004710 1 4
done...
Press any key to run again

Next up comment out the hex dumps...
 
@mjs513 - Good morning.

Probably need to include: UsbMscFat\.....\PFsLib\PFsVolume.h
Which is probably included in some other higher level include files.
Like:
#include "mscFS.h"

Interesting I had tried just including PFsVolume.h but it failed but if I did a #include "mscFS.h" to worked. Live and learn something new everyday.

Ok with that said I added format to the procesMSDrive and it gave me slightly different volume capacities:
Code:
msc # Partition Table
	part,boot,bgnCHS[3],type,endCHS[3],start,length
FAT32:	1,80,0x20,0x21,0x0,0xC,0xC2,0x22,0x20,2048,524288
FAT32:	2,0,0xC2,0x23,0x20,0xC,0xFE,0xFF,0xFF,526336,16564224
exFAT:	3,0,0xFE,0xFF,0xFF,0x7,0xFE,0xFF,0xFF,17090560,228452352
FAT16:	4,0,0xFE,0xFF,0xFF,0xE,0xFE,0xFF,0xFF,245542912,4194304
PFsVolume::begin(20002570)
Partition 0 valid:1
PFsVolume::begin(20002570)
Partition 1 valid:1
PFsVolume::begin(20002570)
Partition 2 valid:1
PFsVolume::begin(20002570)
Partition 3 valid:1
0:>> Fat32: Volume name:(system-boot)###FatPartition::freeClusterCount: FT:32 start:2080 todo:516192
    m_sectorsPerCluster:1
    m_clusterSectorMask:0
    m_sectorsPerClusterShift:0
    m_fatType:32
    m_rootDirEntryCount:0
    m_allocSearchStart:1
    m_sectorsPerFat:4033
    m_dataStartSector:10146
    m_fatStartSector:2080
    m_lastCluster:516191
    m_rootDirStart:2
    m_bytesPerSector:512
[COLOR="#FF0000"] Partition Total Size:264289280 Used:961024 time us: 2123123
    Free Clusters: API: 514313 by CB:514313 time us: 252491
Partition Capacity (MB): 25[/COLOR]6
Format Failed
bench0.txt
bench1.txt
bench2.txt
bench3.txt
bench4.txt
bench5.txt
1:>> Fat32: Volume name:(NEW VOLUME1)###FatPartition::freeClusterCount: FT:32 start:526814 todo:2066434
    m_sectorsPerCluster:8
    m_clusterSectorMask:7
    m_sectorsPerClusterShift:3
    m_fatType:32
    m_rootDirEntryCount:0
    m_allocSearchStart:1
    m_sectorsPerFat:16145
    m_dataStartSector:559104
    m_fatStartSector:526814
    m_lastCluster:2066433
    m_rootDirStart:2
    m_bytesPerSector:512
[COLOR="#FF0000"] Partition Total Size:8464105472 Used:983040 time us: 8452498
    Free Clusters: API: 2066192 by CB:2066192 time us: 1009491
Partition Capacity (MB): 8088[/COLOR]
Format Failed
bench0.txt
bench1.txt
bench2.txt
bench3.txt
bench4.txt
bench5.txt
2:>> ExFat: Volume name:(exFAT-vol3) Partition Total Size:116962361344 Used:786432 time us: 109002
[COLOR="#FF0000"]    Free Clusters: API: 892346 by CB:0 time us: 0
Partition Capacity (MB): 111549[/COLOR]  NOTE this NOT RIGHT for extfat.
Format Failed
3:>> Fat16: Volume name:(FAT16)###FatPartition::freeClusterCount: FT:16 start:245542913 todo:32767
    m_sectorsPerCluster:128
    m_clusterSectorMask:127
    m_sectorsPerClusterShift:7
    m_fatType:16
    m_rootDirEntryCount:512
    m_allocSearchStart:1
    m_sectorsPerFat:128
    m_dataStartSector:245543201
    m_fatStartSector:245542913
    m_lastCluster:32766
    m_rootDirStart:245543169
    m_bytesPerSector:512
[COLOR="#FF0000"] Partition Total Size:0 Used:0 time us: 64000
    Free Clusters: API: 32762 by CB:32762 time us: 8369
Partition Capacity (MB): 2048[/COLOR]  Not sure why total size is zero now.
Format Failed
Partition Capacity (MB): 256
Format Failed

Initialize SD card...initialization failed.

done...
Press any key to run again

EDIT: THINK I AM USING WRONG UPDATED LIBS?
 
Oh = not sure this is right for sector counts:
Code:
  m_sectorCount = getLe32(pt->totalSectors);
 
@KurtE - just did a PR to your Volums_SD branch with what I have so far for FatFormatter. Still playing - like pulling teeth for me - hope I am not too off course.
 
@KurtE - Just got home and downloaded your latest with FsInfo update. If I am seeing this right there is a HUGE difference in the time it takes to get free space on FAT32 formatted partitions:)

This is what I saw on a 32Gig Kingston thumb drive:
Code:
msc # Partition Table
	part,boot,bgnCHS[3],type,endCHS[3],start,length
exFAT:	1,0,0x20,0x21,0x0,0x7,0xFE,0xFF,0xFF,2048,33009664
FAT32:	2,0,0xFE,0xFF,0xFF,0xC,0xFE,0xFF,0xFF,33011712,27424768
pt_#0:	3,0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0,0
pt_#0:	4,0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0,0
PFsVolume::begin(20001ff0)
Partition 0 valid:1
PFsVolume::begin(20001ff0)
Partition 1 valid:1
PFsVolume::begin(20001ff0)
Partition 2 valid:0
PFsVolume::begin(20001ff0)
Partition 3 valid:0
0:>> ExFat: Volume name:(32GEXFAT) Partition Total Size:16898818048 Used:32899072 time us: 63003
    Free Clusters: API: 514707 by CB:0 time us: 0
    Free Clusters: Info: 4294967295 time us: 482
32MEGfile.dat
1:>> Fat32: Volume name:(32GFAT32) Partition Total Size:14027751424 Used:32776192 time us: 5068874
    Free Clusters: API: 1708371 by CB:1708371 time us: [COLOR="#FF0000"]836490[/COLOR]
    Free Clusters: Info: 1708371 time us: [COLOR="#00FF00"]1492[/COLOR]
32MEGfile.dat

I am going to go through FatFS and see what functions updated the FsInfo sector. I am sure it will have a lot to do closing files and unmounting devices. I remember when scanning through 'ff.c' I found it in several places.

Don't think that your C++ programming skills are THAT rusty. Mine are not because I don't have much to rust yet. Still learning. This forum is awesome:)
 
@KurtE - just did a PR to your Volums_SD branch with what I have so far for FatFormatter. Still playing - like pulling teeth for me - hope I am not too off course.
Looks like fun, the PR is in. Will be fun to try formatting some of these devices.

As for being on or off course... Hard for me to say, I am fumbling along as well :D

I assume that since the top level function you pass in: print_t* pr
That many/most of the Serial.print calls in the end will probably be pr->print(...

It will also be interesting to see what level of format support we should try to add: Example the SdFormatter sketch has options:
Code:
  cout << F(
         "\n"
         "Options are:\n"
         "E - erase the card and skip formatting.\n"
         "F - erase and then format the card. (recommended)\n"
         "Q - quick format the card without erase.\n"
         "\n"
         "Enter option: ");
Some of these options I don't fully understand, like format without erase? Unless what they are saying erase is clear out all of the blocks.

I am assuming that it will then call in the SDFat code like SDFormatter.cpp code to then do the main work.

Looks like fun.
 
@KurtE
Morning and back at it.

I noticed the different options - right now just focusing on the "F" option for obvious reasons. One of the things I noticed is that SDFat does the formatting on total sectors so forces a FAT16 even if a 1GB card is formatted for FAT32. Obviously going to make a change to format for whatever format is there :) Right now I am writing to the MBR but have to tweak a few things to see if it works. At least I am at the point where the card is still operational after a format :)

Basically the the format call looks like this:
Code:
FatFormatter.format(msc.usbDrive(), i, partVol[i], sectorBuffer, &Serial);
so pretty much the same except have to pass in the volume and part pointers as well to avoid just formatting the whole card. Still trying to sort out how everything gets hooked together.
 
@mjs513 and @PaulStoffregen @wwatson @defragster (and others) Sounds good. Will be interesting to see what all we can do here. Example when we reformat, can wet set the Volume Label?

As I mentioned elsewhere, wondering again where to integrate things. I think I may take sort of your lead here, and maybe add some of the code we have been playing with in the MSC and MTP sketches into
The PFsVolume Or the like code.

Like the get the volume label.... Maybe add a setVolume Label for those cases.
Maybe some of the get faster free cluster count code... Wish there were some of the defaults in SDFat library be different and/or some of the private member functions be exposed or... But for now probably try to keep as much compatibility as possible...

It is sort of interesting that looked a little at Adafruit's version of SDFat and it looks like they have diverged some like in Fat Partition code it looks like they added support for Fat 1 and Fat 2... Did not look deep enough to see if they were fixing some of the same issues?
 
Follow on to above: pushed up changes to my two forks/branches in MTP and MSC code.
There is now a getVolumeLabel method as part of the PFsVolume class.

Also hacks in place that if you properly cast the BlockDevice interfact to the USB version of it, it uses different begin overload and then remembers the USB blockdevice stuff, such that if you do the call for
freeClusterCount it will use my code to do reads with callbacks to get the values instead of the FatPartition code. Total hack, but then also removed some of the code in the examples.

May also hack in calls for getting the Fat32 Info data for free count and likewise one to update it. Then maybe can have a version of the MTP code that starts off with the saved value so comes up fast, then after things settle and no MTP actions going on, maybe it can use some form of faster version, to verify the cached value. If different maybe it saves it away...

Edit: Also closed out my open issues on SDFat-Beta project
 
Been playing with formatting most of the day and almost have it working. Discovered an issue when I go to multipartition formatted drive. It basically is my lack of understanding advance c++ stuff and making things fit into SDFat library. You probably know that feeling. Not sure if there is an elegant way to fix it or not so going to post it here and see if anyone had any ideas.

In the writeMBR function, SDFAT:
Code:
bool FatFormatter::writeMbr() {
  memset(m_secBuf, 0, BYTES_PER_SECTOR);
  MbrSector_t* mbr = reinterpret_cast<MbrSector_t*>(m_secBuf);

#if USE_LBA_TO_CHS
  lbaToMbrChs(mbr->part->beginCHS, m_capacityMB, m_relativeSectors);
  lbaToMbrChs(mbr->part->endCHS, m_capacityMB,
              m_relativeSectors + m_totalSectors -1);
#else  // USE_LBA_TO_CHS
  mbr->part->beginCHS[0] = 1;
  mbr->part->beginCHS[1] = 1;
  mbr->part->beginCHS[2] = 0;
  mbr->part->endCHS[0] = 0XFE;
  mbr->part->endCHS[1] = 0XFF;
  mbr->part->endCHS[2] = 0XFF;
#endif  // USE_LBA_TO_CHS

  mbr->part->type = m_partType;
  setLe32(mbr->part->relativeSectors, m_relativeSectors);
  setLe32(mbr->part->totalSectors, m_totalSectors);
  setLe16(mbr->signature, MBR_SIGNATURE);
  return m_dev->writeSector(0, m_secBuf);
}
it updates the part based on changes to the MBR for the format selected - always assumes only 1 partition - the whole drive.

When I tried to add change the mbr link to:
Code:
mbr->part[m_part]->beginCHS
I get errors:
Code:
D:\Users\Merli\Documents\Arduino\libraries\UsbMscFat-PFsLib\src\PFsLib\PFsFatFormatter.cpp:289:32: error: base operand of '->' has non-pointer type 'MbrPart_t {aka mbrPartition}'
   lbaToMbrChs(mbr->part[m_part]->beginCHS, m_capacityMB, m_relativeSectors);
                                ^
D:\Users\Merli\Documents\Arduino\libraries\UsbMscFat-PFsLib\src\PFsLib\PFsFatFormatter.cpp:290:20: error: expected unqualified-id before '[' token
   lbaToMbrChs(mbr->[m_part]->endCHS, m_capacityMB,
                    ^
D:\Users\Merli\Documents\Arduino\libraries\UsbMscFat-PFsLib\src\PFsLib\PFsFatFormatter.cpp:301:20: error: base operand of '->' has non-pointer type 'MbrPart_t {aka mbrPartition}'
   mbr->part[m_part]->type = m_partType;
                    ^
D:\Users\Merli\Documents\Arduino\libraries\UsbMscFat-PFsLib\src\PFsLib\PFsFatFormatter.cpp:302:28: error: base operand of '->' has non-pointer type 'MbrPart_t {aka mbrPartition}'
   setLe32(mbr->part[m_part]->relativeSectors, m_relativeSectors);
                            ^
D:\Users\Merli\Documents\Arduino\libraries\UsbMscFat-PFsLib\src\PFsLib\PFsFatFormatter.cpp:303:28: error: base operand of '->' has non-pointer type 'MbrPart_t {aka mbrPartition}'
   setLe32(mbr->part[m_part]->totalSectors, m_totalSectors);
So I changed the function
Code:
bool PFsFatFormatter::writeMbr() {
  memset(m_secBuf, 0, BYTES_PER_SECTOR);
  MbrSector_t* mbr = reinterpret_cast<MbrSector_t*>(m_secBuf);
  
  MbrPart_t *pt = &mbr->part[m_part];
  Serial.println("\nPFsFatFormatter::writeMbr.....");
  Serial.printf("part: %d\n", m_part);

#if USE_LBA_TO_CHS
  lbaToMbrChs(pt->beginCHS, m_capacityMB, m_relativeSectors);
  lbaToMbrChs(pt->endCHS, m_capacityMB,
              m_relativeSectors + m_totalSectors -1);
#else  // USE_LBA_TO_CHS
  pt->beginCHS[0] = 1;
  pt->beginCHS[1] = 1;
  pt->beginCHS[2] = 0;
  pt->endCHS[0] = 0XFE;
  pt->endCHS[1] = 0XFF;
  pt->endCHS[2] = 0XFF;
#endif  // USE_LBA_TO_CHS

  pt->type = m_partType;
  Serial.printf("m_partType: %d\n", pt->type);
  setLe32(pt->relativeSectors, m_relativeSectors);
  setLe32(pt->totalSectors, m_totalSectors);
  setLe16(mbr->signature, MBR_SIGNATURE);
  return m_dev->writeSector(0, m_secBuf);
}
but what happens is that it zeros out the partitions in the MBR that is not updated.

Anyone one have an elegant way of fixing this. If not I think I can loop through the partitions, change the partition I am updating and then continue.

EDIT: There are a few other problems but for now have to get through this one.
 
Back
Top