NativeEthernet stalling with dropped packets

simonwood

Member
I'm sure I'm missing something simple here, but have been investigating this for a few days now and haven't spotted it yet. In the process of migrating a project from Teensy 4.0 with enc28j60 to Teensy 4.1 and native ethernet.

NativeEthernet and FNET are latest from vjmuzik github (today). Arduino environment is 1.8.13, Teensyduino is 1.55

Simple webserver that dumps 25K of text to a web page (mostly based on the webserver example in NativeEthernet) code:

Code:
#include <SPI.h>
#include <NativeEthernet.h>

// Enter a MAC address and IP address for your controller below.
// The IP address will be dependent on your local network:
byte mac[] = {
  0xDE, 0xAD, 0xBE, 0xEF, 0xFE, 0xED
};
IPAddress ip(192, 168, 2, 202);

// Initialize the Ethernet server library
// with the IP address and port you want to use
// (port 80 is default for HTTP):
EthernetServer server(80);



void setup() 
{
  // Open serial communications and wait for port to open:
  Serial.begin(9600);
  while (!Serial) {
    ; // wait for serial port to connect. Needed for native USB port only
  }
  Serial.println("Ethernet WebServer Example");

  // start the Ethernet connection and the server:
  Ethernet.begin(mac, ip);

  // Check for Ethernet hardware present
  if (Ethernet.hardwareStatus() == EthernetNoHardware) 
  {
    Serial.println("Ethernet shield was not found.  Sorry, can't run without hardware. :(");
    while (true) {
      delay(1); // do nothing, no point running without Ethernet hardware
    }
  }
  if (Ethernet.linkStatus() == LinkOFF) 
  {
    Serial.println("Ethernet cable is not connected.");
  }

  // start the server
  server.begin();
  Serial.print("server is at ");
  Serial.println(Ethernet.localIP());
}

void DumpText(EthernetClient& client)
{
  for (int i = 0; i < 250; i++)
  {
    // 102 byte string (crlf)
    client.println("0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789");
  }
  //25k dumped
}

void loop() {
  // listen for incoming clients
  EthernetClient client = server.available();
  if (client) {
    Serial.println("new client");
    // an http request ends with a blank line
    boolean currentLineIsBlank = true;
    while (client.connected()) {
      if (client.available()) {
        char c = client.read();
        Serial.write(c);
        // if you've gotten to the end of the line (received a newline
        // character) and the line is blank, the http request has ended,
        // so you can send a reply
        if (c == '\n' && currentLineIsBlank) {
          // send a standard http response header
          client.println("HTTP/1.1 200 OK");
          client.println("Content-Type: text/html");
          client.println("Connection: close");  // the connection will be closed after completion of the response
          client.println();
          client.println("<!DOCTYPE HTML>");
          client.println("<html>");
          // dump html here
          DumpText(client);
          client.println("</html>");
          break;
        }
        if (c == '\n') {
          // you're starting a new line
          currentLineIsBlank = true;
        } else if (c != '\r') {
          // you've gotten a character on the current line
          currentLineIsBlank = false;
        }
      }
    }
    // give the web browser time to receive the data
    delay(10);
    // close the connection:
    client.close();
    delay(10);
    client.stop();
    Serial.println("client disconnected");
  }
}

Looking in chrome dev view the transfer is stalling and then resuming. I get the same from direct connection or via hub.

Looking with wireshark I see retransmissions occurring a lot (zipped pcap: View attachment ne_dump.zip).

Two different T4.1 boards with different ethernet jacks (one the standard ribbon connected one, the other a custom PCB with magjack) giving same results.

Any ideas gratefully received.

Regards

Simon
 
I'm curious if QNEthernet exhibits the same problem. It's an Ethernet stack with the same [mostly] API but uses a different underlying IP stack (lwIP) and doesn't use timer interrupts for polling RX.

Here's the equivalent code (DHCP; let me know if you want an example with a static IP):
Code:
#include <QNEthernet.h>

using namespace qindesign::network;

constexpr uint32_t kDHCPTimeout = 15000;  // 15 seconds
constexpr uint16_t kServerPort = 80;

// Initialize the Ethernet server library
// with the IP address and port you want to use
// (port 80 is default for HTTP):
EthernetServer server(kServerPort);

void setup() {
  // Open serial communications and wait for port to open
  Serial.begin(115200);
  while (!Serial && millis() < 4000) {
    // Wait for Serial initialization
  }
  Serial.println("Ethernet WebServer Example");

  // Unlike the Arduino API (which you can still use), Ethernet uses
  // the Teensy's internal MAC address by default
  uint8_t mac[6];
  Ethernet.macAddress(mac);
  Serial.printf("MAC = %02x:%02x:%02x:%02x:%02x:%02x\n",
                mac[0], mac[1], mac[2], mac[3], mac[4], mac[5]);

  Serial.println("Starting Ethernet with DHCP...");
  if (!Ethernet.begin()) {
    Serial.println("Failed to start Ethernet");
    return;
  }

  if (!Ethernet.waitForLocalIP(kDHCPTimeout)) {
    Serial.println("Failed to get IP address from DHCP");
  } else {
    IPAddress ip = Ethernet.localIP();
    Serial.printf("    Local IP    = %u.%u.%u.%u\n", ip[0], ip[1], ip[2], ip[3]);
    ip = Ethernet.subnetMask();
    Serial.printf("    Subnet mask = %u.%u.%u.%u\n", ip[0], ip[1], ip[2], ip[3]);
    ip = Ethernet.gatewayIP();
    Serial.printf("    Gateway     = %u.%u.%u.%u\n", ip[0], ip[1], ip[2], ip[3]);
    ip = Ethernet.dnsServerIP();
    Serial.printf("    DNS         = %u.%u.%u.%u\n", ip[0], ip[1], ip[2], ip[3]);

    // Start the server
    Serial.printf("Listening for clients on port %u...\n", kServerPort);
    server.begin();
  }
}

void DumpText(EthernetClient& client) {
  for (int i = 0; i < 250; i++) {
    // 102 byte string (crlf)
    client.println("0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789");
  }
  //25k dumped
}

void loop() {
  // listen for incoming clients
  EthernetClient client = server.available();
  if (client) {
    Serial.println("new client");
    // an http request ends with a blank line
    boolean currentLineIsBlank = true;
    while (client.connected()) {
      if (client.available()) {
        char c = client.read();
        Serial.write(c);
        // if you've gotten to the end of the line (received a newline
        // character) and the line is blank, the http request has ended,
        // so you can send a reply
        if (c == '\n' && currentLineIsBlank) {
          // send a standard http response header
          client.println("HTTP/1.1 200 OK");
          client.println("Content-Type: text/html");
          client.println("Connection: close");  // the connection will be closed after completion of the response
          client.println();
          client.println("<!DOCTYPE HTML>");
          client.println("<html>");
          // dump html here
          DumpText(client);
          client.println("</html>");
          client.flush();
          break;
        }
        if (c == '\n') {
          // you're starting a new line
          currentLineIsBlank = true;
        } else if (c != '\r') {
          // you've gotten a character on the current line
          currentLineIsBlank = false;
        }
      }
    }
    client.stop();  // Stopping because we specified "Connection: close"
    Serial.println("client disconnected");
  }
}

Salient changes:
  1. Don't need SPI.h in QNEthernet.
  2. Don't need to specify MAC (but you can) in QNEthernet.
  3. Changed to DHCP (QNEthernet style).
  4. Added some configuration constants at the top.
  5. Changed serial speed to 115200.
  6. If it got a DHCP address, that means the link is there, so removed Ethernet.linkStatus() check.
  7. Ethernet.hardwareStatus() does nothing in QNEthernet.
  8. Added client.flush() after we're done sending data, but client.stop() will do this anyway later.
  9. client.close() doesn't seem to exist in the Arduino Ethernet API (so I didn't implement it in QNEthernet). stop() is sufficient here.
  10. Shouldn't need the delay because client.stop() waits an amount of time for the connection to be properly flushed and closed. (Data is also flushed in QNEthernet's implementation.)

There's also listeners and such. I plan to add a better HTTP server example to the library at some point.
 
Last edited:
I just updated the `ServerWithAddressListener` example in the library to be a little more complete. See that example for other ways of implementing a server.
 
Well, different.

Using the code you had above I get the same bad stalls (similar dropped packets, timeouts and re-transmits). However, I noticed the amount of data being sent over the cable was wrong.

The DumpText function is trying to put 25K of text into the client for transmission, however it seems that after writing 5752 bytes of it I get a failure back (had to change the code to use client.write so I could get the modified size back). Looping 50 times keeps the data below 5752 bytes and all is good. Going to 100 pushes over and the write returns an error, then I see the retransmission issues.

NativeEthernet has the same transmission issues but transmits all the data (eventually). QNEthernet works great with small packet sizes, but stops and then 'experiences' the same failures as soon as I try to write more than 5752 bytes in that function.

Code:
void DumpText(EthernetClient& client) 
{
  uint32_t nDumped = 0;
  for (int i = 0; i < 50; i++) 
  {
    // 102 byte string (crlf)
    uint16_t nSz = client.write("0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789\n", 101);
    nDumped += nSz;
    if (nSz != 101)
    {
      Serial.println(" failed - ");
      Serial.println(nDumped);
      return;
    }
    //client.println("0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789\n");
  }
  //25k dumped
}
 
Could you add this above setup():
Code:
// QNEthernet links this variable with lwIP's `printf` calls for
// assertions and debugging. User code can also use `printf`.
extern Print *stdPrint;

And this after Serial has been initialized (and before printing "Ethernet WebServer Example"):
Code:
stdPrint = &Serial;  // Make printf work

I'm curious if you're seeing, in the console, the same "large write" errors I've seen. See the "Assertion" To Do in the README. Once you add the above lines (enables `printf` in lwIP and also for user code), do you see something like this:
Assertion "tcp_slowtmr: TIME-WAIT pcb->state == TIME-WAIT" failed at line 1442 in src/lwip/tcp.c

I tried again and noticed the same thing: about 5k being sent (but no assertion). I'll play some more and see if I can manage writing better internally. If I can't find something robust enough, maybe we just have to send data slower, taking into account `client.availableForWrite()` as well.
 
Couple comments:
1. The `print` functions also return the amount of data sent. There's technically no need to switch to `client.write()`, but I support using `write` over `print` because it's "closer to the metal", and some of the `print` functions assume "written fully" behaviour.
2. All output functions that return a "count written" value are meant to behave that way. Returning something less than the amount requested to be sent (in your case 101 bytes), is not an error. It just means the application code needs to retry some of the bytes.

For network applications, I'd shy away from `print`; I’d use `write` and manage "write fully" yourself; `print` does not guarantee "write fully" behaviour. I'm going to add this to the README.

Try changing the DumpText code to this:
Code:
// Keep writing until all the bytes are sent or the connection is closed.
void writeFully(EthernetClient &client, char data[], int len) {
  // Don't use client.connected() as the "connected" check because
  // that will return true if there's data available, and this loop
  // does not check for data available or remove it if it's there.
  while (len > 0 && client) {
    size_t written = client.write(data, len);
    len -= written;
    data += written;
  }
}

void DumpText(EthernetClient& client) {
  static char data102[] = "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789\r\n";
  for (int i = 0; i < 250; i++) {
    writeFully(client, data102, sizeof(data102));
  }
  //25k dumped
}

In summary, for network applications, manage your own writing, taking into account output function return values. Hope this helps.
 
Last edited:
Added those changes. No assertion, but with the changed DumpText it's now transferring the full data set (btw, this is a minimal program to try and reproduce the error rather than something proper ;-) ).

Getting the same stalling effect now it is transmitting all data.

Would be very interesting to know if you get the same.

pcapng file of the transfer: View attachment qne-test.zip load into wireshark and have a look.

Looking at the network traces the QNE library handles MTU chunking much better than the NativeEthernet library. QNE transfers 1514 data bytes per packet, NE only manages that on the first packet and drops to about 154 data bytes per packet for all subsequent interspersed with HTTP continuation packets (you can see that in the pcap in the first post).
 
What do you mean by “stalling”? Is it that all the data gets there but there are periods during the transfer where no traffic is being transmitted? If that is so, which side pauses, or both? When you reload the page, say with a browser, do you notice periods of pausing?

Second question: What happens if you call client.flush() after each call to writeFully()?
 
I'm playing with the code and I don't see any pauses when loading the page in a browser or from the command line, if that's what you mean by "stalling".
 
Are you able to grab a pcap file of your transfer ? You can see the stalls I'm talking about in the pcapng file attached above.

Takes 3-6 seconds to transfer that 25k on my machine. My initial assumption is that I've got something wrong causing that. (actually I haven't tried a new ethernet cable yet - will give that a go later).
 
The transfer is fairly instant for me. I tried with `nc` on the command line and with Safari. The Teensy is connected to an eero and my computer is connected wirelessly to the same network. What does your setup look like?
 
Quick note about client.close() originally I didn't have it and client.stop() behaved in a similar manner to yours does and it did work 99% of the time. After a bit of searching I found that when the server sends a response with "Connection: close" it should immediately close the transmitting side of its connection but it should not stop the socket yet as the client application may still want to send data if the initial request has "Connection: keep-alive" in it. That 1% of the time the computer was sending its own close message while the server was still in the process of doing client.stop() because the computer was expecting the connection to be half closed immediately after the servers last response line. When that happened FNET would respond with a harsh reset and after too many resets my computer would stop even trying to connect to the server. LWIP may not have into this issue, but if it does it you'll just have to implement something similar using tcp_shutdown().
 
I just added `EthernetClient::closeOutput()`, which performs a half close. I also updated the `ServerWithAddressListener` example to use it.

Here's new code (update to the latest QNEthernet on GitHub):
Code:
#include <QNEthernet.h>

using namespace qindesign::network;

constexpr uint32_t kDHCPTimeout = 15000;  // 15 seconds
constexpr uint16_t kServerPort = 80;

// Initialize the Ethernet server library
// with the IP address and port you want to use
// (port 80 is default for HTTP):
EthernetServer server(kServerPort);

// QNEthernet links this variable with lwIP's `printf` calls for
// assertions and debugging. User code can also use `printf`.
extern Print *stdPrint;

void setup() {
  // Open serial communications and wait for port to open
  Serial.begin(115200);
  while (!Serial && millis() < 4000) {
    // Wait for Serial initialization
  }
  stdPrint = &Serial;
  Serial.println("Ethernet WebServer Example");

  // Unlike the Arduino API (which you can still use), Ethernet uses
  // the Teensy's internal MAC address by default
  uint8_t mac[6];
  Ethernet.macAddress(mac);
  Serial.printf("MAC = %02x:%02x:%02x:%02x:%02x:%02x\n",
                mac[0], mac[1], mac[2], mac[3], mac[4], mac[5]);

  Serial.println("Starting Ethernet with DHCP...");
  if (!Ethernet.begin()) {
    Serial.println("Failed to start Ethernet");
    return;
  }

  if (!Ethernet.waitForLocalIP(kDHCPTimeout)) {
    Serial.println("Failed to get IP address from DHCP");
  } else {
    IPAddress ip = Ethernet.localIP();
    Serial.printf("    Local IP    = %u.%u.%u.%u\n", ip[0], ip[1], ip[2], ip[3]);
    ip = Ethernet.subnetMask();
    Serial.printf("    Subnet mask = %u.%u.%u.%u\n", ip[0], ip[1], ip[2], ip[3]);
    ip = Ethernet.gatewayIP();
    Serial.printf("    Gateway     = %u.%u.%u.%u\n", ip[0], ip[1], ip[2], ip[3]);
    ip = Ethernet.dnsServerIP();
    Serial.printf("    DNS         = %u.%u.%u.%u\n", ip[0], ip[1], ip[2], ip[3]);

    // Start the server
    Serial.printf("Listening for clients on port %u...\n", kServerPort);
    server.begin();
  }
}

void DumpText(EthernetClient& client) {
  for (int i = 0; i < 250; i++) {
    // 102 byte string (crlf)
    client.writeFully("0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789\r\n");
  }
  //25k dumped
}

void loop() {
  // listen for incoming clients
  EthernetClient client = server.available();
  if (client) {
    Serial.println("new client");
    // an http request ends with a blank line
    boolean currentLineIsBlank = true;
    while (client.connected()) {
      if (client.available()) {
        char c = client.read();
        Serial.write(c);
        // if you've gotten to the end of the line (received a newline
        // character) and the line is blank, the http request has ended,
        // so you can send a reply
        if (c == '\n' && currentLineIsBlank) {
          // send a standard http response header
          client.println("HTTP/1.1 200 OK");
          client.println("Content-Type: text/html");
          client.println("Connection: close");  // the connection will be closed after completion of the response
          client.println();
          client.println("<!DOCTYPE HTML>");
          client.println("<html>");
          // dump html here
          DumpText(client);
          client.println("</html>");
          client.flush();
          break;
        }
        if (c == '\n') {
          // you're starting a new line
          currentLineIsBlank = true;
        } else if (c != '\r') {
          // you've gotten a character on the current line
          currentLineIsBlank = false;
        }
      }
    }
    client.closeOutput();  // Half closing because we specified "Connection: close"
    // Note: Delays aren't really appropriate here; should track connections with timeouts.
    //       See ServerWithAddressListener example.
    delay(1000);  // Give client a chance to process data
    client.stop();
    Serial.println("client disconnected");
  }
}
 
Back
Top