Teensy 4.1, Native Ethernet, MQTT and a 2 hour mystery

jonp

Member
The story:
I have two Teensy 4.1 units using the onboard Ethernet to communicate with a MQTT server. They are set up fairly similar in the respect that the code is very similar and that they do the same thing - reporting sensor information via MQTT.

Each and by itself the units apparently performs flawlessly. But when i hook up both units in the network, things start to become a little more mysterious than I have so far been able to wrap my brain around...
  • The first unit connected stops working (hangs).
  • The second unit connected works flawlessly for 2hours - to the second, and then stop working.
  • But then the first one starts to work again - and works for, you might guess - two hours.
  • And they go on taking their turn doing their own stuff every two hours.
The question of this post:
Have anyone got any idea what this might come from or where I should start to look...?

Here are some factual information and things I have done so far:
  • The units have different IP-addresses and different MAC-addresses. This is verified by pinging each unit from a Windows PC and then running 'arp -a' to verify MAC-adresses - everything checks out ok.
  • I have a genuine Arduino due with a genuine Arduino Ethernet shield running close to the same setup. This unit does not seem to be affected since I can connect/disconnect it at my will without any misbehavior from this or the two Teensys i question.
  • I am using Arduino 1.8.13 and Teensyduino 1.53.
  • For Ethernet I am using NativeEthernet connected via the onboard interface on the Teensy 4.1
  • The sensors are widely different but both uses LOW/HIGH on GPIO-pins to transfer their knowledge - no external libraries are used.
  • For MQTT i am using PubSubClient 2.8.0
  • MQTT-server is mosquitto running on debian 10 - no authentication or TLS is used.
  • The units do not use the same MQTT-topic and only one of them subscribe to a publisher that publishes only 2 to 3 times a month.
  • The setup has been tried on the same switch and trough up to three switches (Cisco PoE, HP Procurve) - same result.
  • Any type of packet sniffing is not tried - yet (mostly because of the (apparent) hassle of sniffing from a third machine trough a switch)
  • What have I forgot to mention...?
Enclosed is the code for one of the units.
Mentionable things:
  • The network code is mostly cut and paste between the units - ie. it is close to identical. Only IP- an MAC-addresses are different.
  • The overall layout of the program is also identical - down to the "I am Alive" led-stuff at the bottom
  • In this particular code there is an obscene amount of Serial.Prints for bugtracking (which might generate a new post in the near future) - but has no effect on the behavior investigated in this post (I guess/think/hope...).
Code:
#include <NativeEthernet.h>
#include <PubSubClient.h>

char mqttSERVER[] = "192.168.10.11";
const char* mqttServer = mqttSERVER;
IPAddress serv(192, 168, 10, 11);

EthernetClient NetClient;

byte mac[] = {
  0xDE, 0xAD, 0xBE, 0xEF, 0xFE, 0xE3
};
IPAddress ip(192, 168, 10, 165);

void net_setup()
{
  Serial.println("Starting Ethernet");
  Ethernet.begin(mac, ip);
  Serial.println("Checking Ethernet Hardware");
  if (Ethernet.hardwareStatus() == EthernetNoHardware) {
    Serial.println("**** Ethernet shield was not found.  Sorry, can't run without hardware. :(");
    while (true) {
      delay(1); // do nothing, no point running without Ethernet hardware
    }
  }
  Serial.println("Checking Ethernet link");
  if (Ethernet.linkStatus() == LinkOFF) {
    Serial.println("**** Ethernet cable is not connected.");
  }
  Serial.println("Some sort of link...");
}

PubSubClient mqttClient(NetClient);
bool connected = false;

void mqtt_reconnect()
{
  while (!mqttClient.connected()) {
    Serial.print ("Kobler til MQTT-server... ");
    if (mqttClient.connect("teensy")) {
      Serial.println ("Koblet til!");
      //      mqttClient.subscribe("bergen/strom/set");
      connected = true;
    }
    else {
      Serial.println ("**** Ikke koblet til!");
      connected = false;
      delay (1000);
    }
  }
}

void mqtt_loop()
{
  if (!mqttClient.connected()) {
    mqtt_reconnect();
  } else {
    mqttClient.loop();
  }
}

int Humidity = 0;
int Temp = 0;
bool DHTError = false; // Checksum Error

void DHT11(int pin) {
  //  Serial.println ("DHT11 - Read");

  long int DataTime = 0;

  unsigned long result[45];
  int antresult = 0;
  byte data[5];

  // read the Bits and put them into a Result array (It will count 42 bits. The first two one are useless due my code)

  unsigned long prevtime;
  elapsedMicros micro = 0;
  bool block = false;
  //noInterrupts();
  do{
    if (digitalRead(pin) == 0 && block == false){
      result[antresult] = micro - prevtime;
      prevtime = micro;
      block = true;
      antresult++;
    }
    if (digitalRead(pin) == 1)
      block = false;
  }while (micro - prevtime < 250);
  //interrupts();

  // Asign 1 or 0 to Result variable. If more than 80uS Data as "1"
  // Starting at Data set 02. First two Datasets are ignored!

  for (int  i = 2; i < antresult; i++) {
    Serial.print (result[i]);
    Serial.print (" ");
    if (result[i] <= 90)
      result[i] = 0;
    else
      result[i] = 1;
  }
  Serial.println();

  for (int  j = 0; j < 5; j++) { // redo it for the 5 Bytes (40 Databits /8 = 5)
    data[j] = 0;
    for (int  i = 0; i < 8; i++) {
      bitWrite(data[j], 7 - i, result[i + 2 + (j * 8)]);
    }  // Create 5 Databytes from the 40 Databits (Ignoring the 2 first Databits)

  }
  // check checksum                            }

    Serial.print (pin);
    Serial.print ("  ");
    Serial.print (data[0]);
    Serial.print ("  ");
    Serial.print (data[1]);
    Serial.print ("  ");
    Serial.print (data[2]);
    Serial.print ("  ");
    Serial.print (data[3]);
    Serial.print ("  ");
    Serial.print (data[4]);
  if (data[4] == (data[0] + data[1] + data[2] + data[3])) {
    Humidity = data[0] * 10 + data[1];
    Temp = data[2] * 10 + data[3];
    DHTError = false;
  }
  else {
    DHTError = true; //If Checksum is worng, Temp=99 (Dataset 0-3 in addition = Dataset 4 = Checksum OK)
    Humidity = 990;
    Temp = 950;
    Serial.print ("  Error!");
  }
    Serial.println();
}

char str[] = "{\"hum\":XX.X,"
             "\"temp\":XX.X,"
             "\"id\":X,"
             "\"error\":X}";

elapsedMillis blk_live;
elapsedMillis intervall = 0;

int DHTPorts[] = {3, 4, 0};
int DHTActive[] = {1, 1, -1};
bool DHTStart = false;

elapsedMicros DHTSeq;

int DHTState = -1;
int DHTPin;
int DHTIndex = -1;

#define LED_LIVE 13

void setup() {
  Serial.begin(115200);
  pinMode(LED_LIVE, OUTPUT);
  digitalWrite(LED_LIVE, HIGH);

  delay(2000);
  digitalWrite(LED_LIVE, LOW);
  Serial.println ("Net Setup");
  net_setup();
  digitalWrite(LED_LIVE, HIGH);
  delay(500);
  mqttClient.setServer(serv, 1883);
}

void loop() {

  if (intervall > 5050) {
    intervall = 0;
    if (connected == true) {
      DHTStart = true;
      DHTIndex++;
      if (DHTActive [DHTIndex] == -1)
        DHTIndex = 0;
    }
    Serial.println ("Intervall");
  }

  if (DHTStart) {
    DHTStart = false;
    DHTState = 5;
  }

  if (DHTActive [DHTIndex] == 1 && DHTState == 5) {
    DHTState = 0;
    DHTSeq = 0;
    DHTPin = DHTPorts[DHTIndex];
  }

  if (DHTState == 0) {
    pinMode(DHTPin, OUTPUT);
    digitalWrite(DHTPin, HIGH);
    DHTState ++;
  }
  if (DHTState == 1 && DHTSeq > 250000) {
    digitalWrite(DHTPin, LOW);
    DHTState++;
  }
  if (DHTState == 2 && DHTSeq > 280000) {
    digitalWrite(DHTPin, HIGH);
    DHTState++;
  }
  if (DHTState == 3 && DHTSeq > 280050) {
    pinMode(DHTPin, INPUT);
    DHTState++;
  }

  if (DHTState == 4) {
    DHTState = 6;

    DHT11(DHTPin);

    int h, t;
    h = Humidity;
    t = Temp;

    Serial.print ("Hum : ");
    Serial.print (h);
    Serial.print ("  Temp : ");
    Serial.println (t);

    char ctmp;
    ctmp = h / 100;
    str[7] = ctmp + 48;
    h -= ctmp * 100;
    ctmp = h / 10;
    str[8] = ctmp + 48;
    h -= ctmp * 10;
    str[10] = h + 48;

    ctmp = t / 100;
    str[19] = ctmp + 48;
    t -= ctmp * 100;
    ctmp = t / 10;
    str[20] = ctmp + 48;
    t -= ctmp * 10;
    str[22] = t + 48;
    str[29] = DHTIndex + 48;

    if (DHTError)
      str[39] = '1';
    else
      str[39] = '0';

    mqttClient.publish ("bergen/fukt/kjeller", str, 29);
    //    Serial.println("Da er ting sendt avgårde");
  }

  mqtt_loop();

  if (blk_live > 250) {
    digitalWrite(LED_LIVE, LOW);
  }
  if (blk_live > 500) {
    digitalWrite(LED_LIVE, HIGH);
    blk_live = 0;
  }
}
 
Try the updates shown here, see if they fix your issue, there was an issue of running out of sockets when using TCP Servers for a while now. This is more than likely what is happening to you since as you say it was hanging.

I've finally dedicated some time to hopefully fixing this and I may have done it though some more testing should be done to catch any edge cases that could be happening and verifying that no data is being lost like it used to be.

For NativeEthernet I've already made a commit because I've previously tested this fix when I first looked into this, it stops NativeEthernet from actually running out of sockets and completely stopping when FNETs TCP code doesn't behave like you would expect it too.
https://github.com/vjmuzik/NativeEthernet

As for FNET, I've narrowed it down to where the data was getting thrown out though I haven't figured out why exactly since as far as WireShark can tell the packet that was getting thrown out was the same as previous ones that had no issues. So I've commented out the lines where it would close the socket that was making NativeEthernet have issues and as far as I can tell with Apache Benchmark there doesn't seem to be any missed packets or dropped connections like there used to be. If this fix works for everyone without causing any side effects I will go ahead and merge it.
https://github.com/vjmuzik/FNET/tree/TCPListenPatch-1

Code for testing:
Code:
#include "NativeEthernet.h"

uint8_t mac[6];
void teensyMAC(uint8_t *mac) {
    for(uint8_t by=0; by<2; by++) mac[by]=(HW_OCOTP_MAC1 >> ((1-by)*8)) & 0xFF;
    for(uint8_t by=0; by<4; by++) mac[by+2]=(HW_OCOTP_MAC0 >> ((3-by)*8)) & 0xFF;
    Serial.printf("MAC: %02x:%02x:%02x:%02x:%02x:%02x\n", mac[0], mac[1], mac[2], mac[3], mac[4], mac[5]);
}

//EthernetServer server(443, true); //Uncomment for TLS
EthernetServer server(80);

void setup() {;
  // put your setup code here, to run once:
  teensyMAC(mac);
  Ethernet.begin(mac);
  MDNS.begin("Teensy41", 2); //.local Domain name and number of services
//  MDNS.setServiceName("Teensy41_Service_Name"); //Uncomment to change service name
//  MDNS.addService("_https._tcp", 443); //Uncomment for TLS
  MDNS.addService("_http._tcp", 80);
  server.begin();
  Serial.print("IP  address: ");
  Serial.println(Ethernet.localIP());
  Serial.send_now();
}

void loop() {
  // listen for incoming clients
  EthernetClient client = server.available();
  if (client) {
    Serial.println("new client");
    // an http request ends with a blank line
    boolean currentLineIsBlank = true;
    while (client.connected()) {
      if (client.available()) {
        char c = client.read();
        Serial.write(c);
        // if you've gotten to the end of the line (received a newline
        // character) and the line is blank, the http request has ended,
        // so you can send a reply
        if (c == '\n' && currentLineIsBlank) {
          // send a standard http response header
          client.println("HTTP/1.1 200 OK");
          client.println("Content-Type: text/html");
          client.println("Connection: close");  // the connection will be closed after completion of the response
//          client.println("Refresh: 1");  // refresh the page automatically every 5 sec
          client.println();
          client.println("<!DOCTYPE HTML>");
          client.println("<html>");
          // output the value of each analog input pin
          for (int analogChannel = 0; analogChannel < 6; analogChannel++) {
            int sensorReading = analogRead(analogChannel);
            client.print("analog input ");
            client.print(analogChannel);
            client.print(" is ");
            client.print(sensorReading);
            client.println("<br />");
          }
          client.println("</html>");
          client.close(); //If "Connection: close" make sure to close after printing and before stop to avoid harsh reset
          break;
        }
        if (c == '\n') {
          // you're starting a new line
          currentLineIsBlank = true;
        } else if (c != '\r') {
          // you've gotten a character on the current line
          currentLineIsBlank = false;
        }
      }
    }
    // give the web browser time to receive the data
    delay(1);
    // close the connection:
    client.stop();
    Serial.println("client disconnected");
  }
}

Apache Benchmark Command
Code:
ab -k -c 10 -n 10000 http://teensy41.local/
 
An alternative to updating the libraries manually is to install TeensyDuino 1.55 Beta 1, which contains the new versions of FNET and NativeEthernet.
 
An alternative to updating the libraries manually is to install TeensyDuino 1.55 Beta 1, which contains the new versions of FNET and NativeEthernet.

Thanks for the inputs,

I tried the tip quoted above - with some quirky results...
  • I installed 1.55 beta1 over the already existing installation ( I forgot to mention that Arduino is running on Debian 10 in a Virtualbox VM which in turn is running on Windows 10)
  • I reprogrammed both units mentioned in the original post.
  • I hooked them both up in the network
The results are approximately as follows:
  • When one unit is running by itself it seems to work as expected
  • Now, when I hook up the other unit, both units starts to drop packets - in the sense that software using the MQTT-messages does not get all the messages. This has not happened before.
  • When i disconnect one of the units, the other resumes business as usual and performs as expected - no messages dropped.
  • Without being too certain, it might seem that messages are lost while the other unit is already transmitting.
What bothers me a bit is that the units does not seem to like each other. I have no idea on which layer (think OSI-model) this malcontent is happening. If they had identical IP-addresses and/or identical MAC-addresses I could understand such behavior, but I am pretty sure that is not the case.<

in the meantime all other computers and stuff on the net is working as they should - including the Arduino due mentioned in my original post.
 
Last edited:
I don’t know what versions are included in the beta, I would still recommend trying the manual install first from the provided links.
 
I don’t know what versions are included in the beta, I would still recommend trying the manual install first from the provided links.

I will look into it when I get home.

In the meantime, do you have any idea why they don't work when they are two T4.1's on the net, but work without hickups when there is only one...?
 
Have you tried packet sniffing with Wireshark via a monitor port? Either with a basic hub or managed switch? This could provide some insight.
 
Have you tried packet sniffing with Wireshark via a monitor port? Either with a basic hub or managed switch? This could provide some insight.

As I mentioned in my OP I have not done that, yet - mostly because the things that you mention: I don't have a hub and the desperation to overcome the tedium of reprogramming my rather old Cisco 3750 have yet to disturb my sleep. But I am working on it - mobilizing the desperation that is...
 
Hi I am trying to connect a Teensy 4.1 to a Mosquitto MQTT broker running on my local PC
the broker name is localhost and is on port 1883
I am using latest versions of the NativeEthernet library and the PubSubClient library
this is my code( its just the mqtt_basic example given in the pubSubClient library)
Code:
#include <SPI.h>
#include <NativeEthernet.h>
#include <PubSubClient.h>

// Update these with values suitable for your network.
byte mac[6];
IPAddress ip(172, 16, 0, 100);
//IPAddress server(142, 66, 215, 171);
const char* server = "localhost";
void teensyMAC(uint8_t *mac);
void callback(char* topic, byte* payload, unsigned int length) {
  Serial.print("Message arrived [");
  Serial.print(topic);
  Serial.print("] ");
  for (int i=0;i<length;i++) {
    Serial.print((char)payload[i]);
  }
  Serial.println();
}

EthernetClient ethClient;
PubSubClient client(ethClient);

void reconnect() {
  // Loop until we're reconnected
  while (!client.connected()) {
    Serial.print("Attempting MQTT connection....");
    // Attempt to connect
    if (client.connect("WeatherStation")) {
      Serial.println("connected");
      // Once connected, publish an announcement...
      client.publish("Sensor","hello world");
      // ... and resubscribe
      //client.subscribe("inTopic");
    } else {
      Serial.print("failed, rc=");
      Serial.print(client.state());
      Serial.println(" try again in 5 seconds");
      // Wait 5 seconds before retrying
      delay(5000);
    }
  }
}

void setup()
{
  Serial.begin(57600);
  teensyMAC(mac);
  if (Ethernet.begin(mac) == 0) {
    Serial.println("Failed to configure Ethernet using DHCP");
    // Check for Ethernet hardware present
    if (Ethernet.hardwareStatus() == EthernetNoHardware) {
      Serial.println("Ethernet shield was not found.  Sorry, can't run without hardware. :(");
      while (true) {
        delay(1); // do nothing, no point running without Ethernet hardware
      }
    }
    if (Ethernet.linkStatus() == LinkOFF) {
      Serial.println("Ethernet cable is not connected.");
    }
  }
  else
  {Serial.println("Success");
  Serial.print("Assigned IP ");
  Serial.println(Ethernet.localIP());
  }
  
  client.setServer(server, 1883);
  client.setCallback(callback);

  //Ethernet.begin(mac, ip);
  // Allow the hardware to sort itself out
  delay(1500);
}

void loop()
{
  if (!client.connected()) {
    reconnect();
  }
  client.loop();
}

void teensyMAC(uint8_t *mac) 
{for(uint8_t by=0; by<2; by++) mac[by]=(HW_OCOTP_MAC1 >> ((1-by)*8)) & 0xFF;
 for(uint8_t by=0; by<4; by++) mac[by+2]=(HW_OCOTP_MAC0 >> ((3-by)*8)) & 0xFF;
 Serial.printf("MAC: %02x:%02x:%02x:%02x:%02x:%02x\n", mac[0], mac[1], mac[2], mac[3], mac[4], mac[5]);
}

The teensy gets an assigned IP address of 142.66.211.194 and I can ping from my pc to the address
however when the teensy tries to connect to MQTT it keeps getting 'Attempting MQTT connection....failed, rc=-2 try again in 5 seconds'
 
Back
Top