usbMIDI transmission error

Here is the solution that works for me on T3.6. Rather than commenting out the midi_flush_output line as suggested before, add a variable at the top of the usb_midi.c that is used to notify the flush routine that a sysex write is in progress.

Then modify the two sysex write routines to set that flag while the routines are active. The routines are called usb_midi_send_sysex_buffer_has_term() and usb_midi_send_sysex_add_term_bytes(). After setting the flag back to 0 at the end of those routines, add a call to the flush routine usb_midi_flush_output() so you don't need to do this in your own program after each sysex buffer write.

Lastly, modify usb_midi_flush_output to check the flag is not set before continuing with any processing.

Here is my modified version of usb_midi.c for Teensy 3, but a simlar approach should work for T4. In the code, the lines that have been added have my initials (rvh) as a comment so you can see where I've made changes

Code: 
/* Teensyduino Core Library
 * http://www.pjrc.com/teensy/
 * Copyright (c) 2017 PJRC.COM, LLC.
 *
 * Permission is hereby granted, free of charge, to any person obtaining
 * a copy of this software and associated documentation files (the
 * "Software"), to deal in the Software without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Software, and to
 * permit persons to whom the Software is furnished to do so, subject to
 * the following conditions:
 *
 * 1. The above copyright notice and this permission notice shall be
 * included in all copies or substantial portions of the Software.
 *
 * 2. If the Software is incorporated into a build system that allows
 * selection among a list of target devices, then similar target
 * devices manufactured by PJRC.COM must be included in the list of
 * target devices and selectable in the same manner.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

#include "usb_dev.h"
#include "usb_midi.h"
#include "core_pins.h" // for yield()
#include "HardwareSerial.h"

#ifdef MIDI_INTERFACE // defined by usb_dev.h -> usb_desc.h
#if F_CPU >= 20000000

uint8_t usb_midi_msg_cable;
uint8_t usb_midi_msg_channel;
uint8_t usb_midi_msg_type;
uint8_t usb_midi_msg_data1;
uint8_t usb_midi_msg_data2;
// TODO: separate sysex buffers for each cable...
uint8_t usb_midi_msg_sysex[USB_MIDI_SYSEX_MAX];
uint16_t usb_midi_msg_sysex_len;
void (*usb_midi_handleNoteOff)(uint8_t ch, uint8_t note, uint8_t vel) = NULL;
void (*usb_midi_handleNoteOn)(uint8_t ch, uint8_t note, uint8_t vel) = NULL;
void (*usb_midi_handleVelocityChange)(uint8_t ch, uint8_t note, uint8_t vel) = NULL;
void (*usb_midi_handleControlChange)(uint8_t ch, uint8_t control, uint8_t value) = NULL;
void (*usb_midi_handleProgramChange)(uint8_t ch, uint8_t program) = NULL;
void (*usb_midi_handleAfterTouch)(uint8_t ch, uint8_t pressure) = NULL;
void (*usb_midi_handlePitchChange)(uint8_t ch, int pitch) = NULL;
void (*usb_midi_handleSysExPartial)(const uint8_t *data, uint16_t length, uint8_t complete) = NULL;
void (*usb_midi_handleSysExComplete)(uint8_t *data, unsigned int size) = NULL;
void (*usb_midi_handleTimeCodeQuarterFrame)(uint8_t data) = NULL;
void (*usb_midi_handleSongPosition)(uint16_t beats) = NULL;
void (*usb_midi_handleSongSelect)(uint8_t songnumber) = NULL;
void (*usb_midi_handleTuneRequest)(void) = NULL;
void (*usb_midi_handleClock)(void) = NULL;
void (*usb_midi_handleStart)(void) = NULL;
void (*usb_midi_handleContinue)(void) = NULL;
void (*usb_midi_handleStop)(void) = NULL;
void (*usb_midi_handleActiveSensing)(void) = NULL;
void (*usb_midi_handleSystemReset)(void) = NULL;
void (*usb_midi_handleRealTimeSystem)(uint8_t rtb) = NULL;


// Maximum number of transmit packets to queue so we don't starve other endpoints for memory
#define TX_PACKET_LIMIT 6
static usb_packet_t *rx_packet=NULL;
static usb_packet_t *tx_packet=NULL;
static uint8_t transmit_previous_timeout=0;
static uint8_t tx_noautoflush=0;

static uint8_t SysExInProgress = 0;  //  rvh

// When the PC isn't listening, how long do we wait before discarding data?
#define TX_TIMEOUT_MSEC 40
#if F_CPU == 256000000
  #define TX_TIMEOUT (TX_TIMEOUT_MSEC * 1706)
#elif F_CPU == 240000000
  #define TX_TIMEOUT (TX_TIMEOUT_MSEC * 1600)
#elif F_CPU == 216000000
  #define TX_TIMEOUT (TX_TIMEOUT_MSEC * 1440)
#elif F_CPU == 192000000
  #define TX_TIMEOUT (TX_TIMEOUT_MSEC * 1280)
#elif F_CPU == 180000000
  #define TX_TIMEOUT (TX_TIMEOUT_MSEC * 1200)
#elif F_CPU == 168000000
  #define TX_TIMEOUT (TX_TIMEOUT_MSEC * 1100)
#elif F_CPU == 144000000
  #define TX_TIMEOUT (TX_TIMEOUT_MSEC * 932)
#elif F_CPU == 120000000
  #define TX_TIMEOUT (TX_TIMEOUT_MSEC * 764)
#elif F_CPU == 96000000
  #define TX_TIMEOUT (TX_TIMEOUT_MSEC * 596)
#elif F_CPU == 72000000
  #define TX_TIMEOUT (TX_TIMEOUT_MSEC * 512)
#elif F_CPU == 48000000
  #define TX_TIMEOUT (TX_TIMEOUT_MSEC * 428)
#elif F_CPU == 24000000
  #define TX_TIMEOUT (TX_TIMEOUT_MSEC * 262)
#endif


// This 32 bit input format is documented in the "Universal Serial Bus Device Class
// Definition for MIDI Devices" specification, version 1.0, Nov 1, 1999.  It can be
// downloaded from www.usb.org.  https://www.usb.org/sites/default/files/midi10.pdf
// If the USB-IF reorganizes their website and this link no longer works, Google
// search the name to find it.  This data format is shown on page 16 in Figure #8.
// Byte 0 (shown on the left hand side of Figure #8) is the least significant byte
// of this 32 bit input.
void usb_midi_write_packed(uint32_t n)
{
	uint32_t index, wait_count=0;

	tx_noautoflush = 1;
	if (!tx_packet) {
        	while (1) {
                	if (!usb_configuration) {
				//serial_print("error1\n");
                        	return;
                	}
                	if (usb_tx_packet_count(MIDI_TX_ENDPOINT) < TX_PACKET_LIMIT) {
                        	tx_packet = usb_malloc();
                        	if (tx_packet) break;
                	}
                	if (++wait_count > TX_TIMEOUT || transmit_previous_timeout) {
                        	transmit_previous_timeout = 1;
				//serial_print("error2\n");
                        	return;
                	}
                	yield();
        	}
	}
	transmit_previous_timeout = 0;
	index = tx_packet->index;
	((uint32_t *)(tx_packet->buf))[index++] = n;
	if (index < MIDI_TX_SIZE/4) {
		tx_packet->index = index;
	} else {
		tx_packet->len = MIDI_TX_SIZE;
		usb_tx(MIDI_TX_ENDPOINT, tx_packet);
		tx_packet = NULL;
	}
	tx_noautoflush = 0;
}

void usb_midi_send_sysex_buffer_has_term(const uint8_t *data, uint32_t length, uint8_t cable)
{
     SysExInProgress = 1;            // rvh
	
        cable = (cable & 0x0F) << 4;
        while (length > 3) {
                usb_midi_write_packed(0x04 | cable | (data[0] << 8) | (data[1] << 16) | (data[2] << 24));
                data += 3;
                length -= 3;
        }
        if (length == 3) {
                usb_midi_write_packed(0x07 | cable | (data[0] << 8) | (data[1] << 16) | (data[2] << 24));
        } else if (length == 2) {
                usb_midi_write_packed(0x06 | cable | (data[0] << 8) | (data[1] << 16));
        } else if (length == 1) {
                usb_midi_write_packed(0x05 | cable | (data[0] << 8));
        }

	SysExInProgress = 0;            // rvh
     usb_midi_flush_output();

}

void usb_midi_send_sysex_add_term_bytes(const uint8_t *data, uint32_t length, uint8_t cable)
{
	SysExInProgress = 1;            // rvh
	cable = (cable & 0x0F) << 4;

	if (length == 0) {
		usb_midi_write_packed(0x06 | cable | (0xF0 << 8) | (0xF7 << 16));
		return;
	} else if (length == 1) {
		usb_midi_write_packed(0x07 | cable | (0xF0 << 8) | (data[0] << 16) | (0xF7 << 24));
		return;
	} else {
		usb_midi_write_packed(0x04 | cable | (0xF0 << 8) | (data[0] << 16) | (data[1] << 24));
		data += 2;
		length -= 2;
	}
	while (length >= 3) {
		usb_midi_write_packed(0x04 | cable | (data[0] << 8) | (data[1] << 16) | (data[2] << 24));
		data += 3;
		length -= 3;
	}
	if (length == 2) {
		usb_midi_write_packed(0x07 | cable | (data[0] << 8) | (data[1] << 16) | (0xF7 << 24));
	} else if (length == 1) {
                usb_midi_write_packed(0x06 | cable | (data[0] << 8) | (0xF7 << 16));
	} else {
                usb_midi_write_packed(0x05 | cable | (0xF7 << 8));
	}
     SysExInProgress = 0;            // rvh
     usb_midi_flush_output();
}

void usb_midi_flush_output(void)
{
	if(SysExInProgress == 0)            // rvh
	if (tx_noautoflush == 0) {
		tx_noautoflush = 1;
		if (tx_packet && tx_packet->index > 0) {
			tx_packet->len = tx_packet->index * 4;
			usb_tx(MIDI_TX_ENDPOINT, tx_packet);
			tx_packet = NULL;
		}
		tx_noautoflush = 0;
	}
}

void static sysex_byte(uint8_t b)
{
	if (usb_midi_handleSysExPartial && usb_midi_msg_sysex_len >= USB_MIDI_SYSEX_MAX) {
		// when buffer is full, send another chunk to partial handler.
		(*usb_midi_handleSysExPartial)(usb_midi_msg_sysex, usb_midi_msg_sysex_len, 0);
		usb_midi_msg_sysex_len = 0;
	}
	if (usb_midi_msg_sysex_len < USB_MIDI_SYSEX_MAX) {
		usb_midi_msg_sysex[usb_midi_msg_sysex_len++] = b;
	}
}

uint32_t usb_midi_available(void)
{
	uint32_t index;

	if (!rx_packet) {
		if (!usb_configuration) return 0;
		rx_packet = usb_rx(MIDI_RX_ENDPOINT);
		if (!rx_packet) return 0;
		if (rx_packet->len == 0) {
			usb_free(rx_packet);
			rx_packet = NULL;
			return 0;
		}
	}
	index = rx_packet->index;
	return rx_packet->len - index;
}

uint32_t usb_midi_read_message(void)
{
	uint32_t n, index;

	if (!rx_packet) {
		if (!usb_configuration) return 0;
		rx_packet = usb_rx(MIDI_RX_ENDPOINT);
		if (!rx_packet) return 0;
		if (rx_packet->len == 0) {
			usb_free(rx_packet);
			rx_packet = NULL;
			return 0;
		}
	}
	index = rx_packet->index;
	n = ((uint32_t *)rx_packet->buf)[index/4];
	index += 4;
	if (index < rx_packet->len) {
		rx_packet->index = index;
	} else {
		usb_free(rx_packet);
		rx_packet = usb_rx(MIDI_RX_ENDPOINT);
	}
	return n;
}

int usb_midi_read(uint32_t channel)
{
	uint32_t n, index, ch, type1, type2, b1;

	if (!rx_packet) {
		if (!usb_configuration) return 0;
		rx_packet = usb_rx(MIDI_RX_ENDPOINT);
		if (!rx_packet) return 0;
		if (rx_packet->len == 0) {
			usb_free(rx_packet);
			rx_packet = NULL;
			return 0;
		}
	}
	index = rx_packet->index;
	n = ((uint32_t *)rx_packet->buf)[index/4];
	//serial_print("midi rx, n=");
	//serial_phex32(n);
	//serial_print("\n");
	index += 4;
	if (index < rx_packet->len) {
		rx_packet->index = index;
	} else {
		usb_free(rx_packet);
		rx_packet = usb_rx(MIDI_RX_ENDPOINT);
	}
	type1 = n & 15;
	type2 = (n >> 12) & 15;
	b1 = (n >> 8) & 0xFF;
	ch = (b1 & 15) + 1;
	usb_midi_msg_cable = (n >> 4) & 15;
	if (type1 >= 0x08 && type1 <= 0x0E) {
		if (channel && channel != ch) {
			// ignore other channels when user wants single channel read
			return 0;
		}
		if (type1 == 0x08 && type2 == 0x08) {
			usb_midi_msg_type = 0x80;		// 0x80 = usbMIDI.NoteOff
			if (usb_midi_handleNoteOff)
				(*usb_midi_handleNoteOff)(ch, (n >> 16), (n >> 24));
		} else
		if (type1 == 0x09 && type2 == 0x09) {
			if ((n >> 24) > 0) {
				usb_midi_msg_type = 0x90;	// 0x90 = usbMIDI.NoteOn
				if (usb_midi_handleNoteOn)
					(*usb_midi_handleNoteOn)(ch, (n >> 16), (n >> 24));
			} else {
				usb_midi_msg_type = 0x80;	// 0x80 = usbMIDI.NoteOff
				if (usb_midi_handleNoteOff)
					(*usb_midi_handleNoteOff)(ch, (n >> 16), (n >> 24));
			}
		} else
		if (type1 == 0x0A && type2 == 0x0A) {
			usb_midi_msg_type = 0xA0;		// 0xA0 = usbMIDI.AfterTouchPoly
			if (usb_midi_handleVelocityChange)
				(*usb_midi_handleVelocityChange)(ch, (n >> 16), (n >> 24));
		} else
		if (type1 == 0x0B && type2 == 0x0B) {
			usb_midi_msg_type = 0xB0;		// 0xB0 = usbMIDI.ControlChange
			if (usb_midi_handleControlChange)
				(*usb_midi_handleControlChange)(ch, (n >> 16), (n >> 24));
		} else
		if (type1 == 0x0C && type2 == 0x0C) {
			usb_midi_msg_type = 0xC0;		// 0xC0 = usbMIDI.ProgramChange
			if (usb_midi_handleProgramChange)
				(*usb_midi_handleProgramChange)(ch, (n >> 16));
		} else
		if (type1 == 0x0D && type2 == 0x0D) {
			usb_midi_msg_type = 0xD0;		// 0xD0 = usbMIDI.AfterTouchChannel
			if (usb_midi_handleAfterTouch)
				(*usb_midi_handleAfterTouch)(ch, (n >> 16));
		} else
		if (type1 == 0x0E && type2 == 0x0E) {
			usb_midi_msg_type = 0xE0;		// 0xE0 = usbMIDI.PitchBend
			if (usb_midi_handlePitchChange) {
				int value = ((n >> 16) & 0x7F) | ((n >> 17) & 0x3F80);
				value -= 8192; // 0 to 16383 --> -8192 to +8191
				(*usb_midi_handlePitchChange)(ch, value);
			}
		} else {
			return 0;
		}
		return_message:
		usb_midi_msg_channel = ch;
		usb_midi_msg_data1 = (n >> 16);
		usb_midi_msg_data2 = (n >> 24);
		return 1;
	}
	if (type1 == 0x02 || type1 == 0x03 || (type1 == 0x05 && b1 >= 0xF1 && b1 != 0xF7)) {
		// system common or system realtime message
		system_common_or_realtime:
		switch (b1) {
		  case 0xF1: // usbMIDI.TimeCodeQuarterFrame
			if (usb_midi_handleTimeCodeQuarterFrame) {
				(*usb_midi_handleTimeCodeQuarterFrame)(n >> 16);
			}
			break;
		  case 0xF2: // usbMIDI.SongPosition
			if (usb_midi_handleSongPosition) {
				(*usb_midi_handleSongPosition)(
				  ((n >> 16) & 0x7F) | ((n >> 17) & 0x3F80));
			}
			break;
		  case 0xF3: // usbMIDI.SongSelect
			if (usb_midi_handleSongSelect) {
				(*usb_midi_handleSongSelect)(n >> 16);
			}
			break;
		  case 0xF6: // usbMIDI.TuneRequest
			if (usb_midi_handleTuneRequest) {
				(*usb_midi_handleTuneRequest)();
			}
			break;
		  case 0xF8: // usbMIDI.Clock
			if (usb_midi_handleClock) {
				(*usb_midi_handleClock)();
			} else if (usb_midi_handleRealTimeSystem) {
				(*usb_midi_handleRealTimeSystem)(0xF8);
			}
			break;
		  case 0xFA: // usbMIDI.Start
			if (usb_midi_handleStart) {
				(*usb_midi_handleStart)();
			} else if (usb_midi_handleRealTimeSystem) {
				(*usb_midi_handleRealTimeSystem)(0xFA);
			}
			break;
		  case 0xFB: // usbMIDI.Continue
			if (usb_midi_handleContinue) {
				(*usb_midi_handleContinue)();
			} else if (usb_midi_handleRealTimeSystem) {
				(*usb_midi_handleRealTimeSystem)(0xFB);
			}
			break;
		  case 0xFC: // usbMIDI.Stop
			if (usb_midi_handleStop) {
				(*usb_midi_handleStop)();
			} else if (usb_midi_handleRealTimeSystem) {
				(*usb_midi_handleRealTimeSystem)(0xFC);
			}
			break;
		  case 0xFE: // usbMIDI.ActiveSensing
			if (usb_midi_handleActiveSensing) {
				(*usb_midi_handleActiveSensing)();
			} else if (usb_midi_handleRealTimeSystem) {
				(*usb_midi_handleRealTimeSystem)(0xFE);
			}
			break;
		  case 0xFF: // usbMIDI.SystemReset
			if (usb_midi_handleSystemReset) {
				(*usb_midi_handleSystemReset)();
			} else if (usb_midi_handleRealTimeSystem) {
				(*usb_midi_handleRealTimeSystem)(0xFF);
			}
			break;
		  default:
			return 0; // unknown message, ignore it
		}
		usb_midi_msg_type = b1;
		goto return_message;
	}
	if (type1 == 0x04) {
		sysex_byte(n >> 8);
		sysex_byte(n >> 16);
		sysex_byte(n >> 24);
		return 0;
	}
	if (type1 >= 0x05 && type1 <= 0x07) {
		sysex_byte(b1);
		if (type1 >= 0x06) sysex_byte(n >> 16);
		if (type1 == 0x07) sysex_byte(n >> 24);
		uint16_t len = usb_midi_msg_sysex_len;
		usb_midi_msg_data1 = len;
		usb_midi_msg_data2 = len >> 8;
		usb_midi_msg_sysex_len = 0;
		usb_midi_msg_type = 0xF0;			// 0xF0 = usbMIDI.SystemExclusive
		if (usb_midi_handleSysExPartial) {
			(*usb_midi_handleSysExPartial)(usb_midi_msg_sysex, len, 1);
		} else if (usb_midi_handleSysExComplete) {
			(*usb_midi_handleSysExComplete)(usb_midi_msg_sysex, len);
		}
		return 1;
	}
	if (type1 == 0x0F) {
		if (b1 >= 0xF8) {
			// From Sebastian Tomczak, seb.tomczak at gmail.com
			// http://little-scale.blogspot.com/2011/08/usb-midi-game-boy-sync-for-16.html
			goto system_common_or_realtime;
		}
		if (b1 == 0xF0 || usb_midi_msg_sysex_len > 0) {
			// From David Sorlien, dsorlien at gmail.com, http://axe4live.wordpress.com
			// OSX sometimes uses Single Byte Unparsed to
			// send bytes in the middle of a SYSEX message.
			sysex_byte(b1);
		}
	}
	return 0;
}


#endif // F_CPU
#endif // MIDI_INTERFACE
 
Can you help me to reproduce this problem?

Before I edit the USB code for everyone (eg, Teensyduino 1.59...), I want to watch the actual USB communication with my USB protocol analyzer to really see what's going on.

Please understand I'm not a musician or synth person. I also don't use Windows much. I see 2 test programs, in msg #30 and msg #46. But neither has a clear explanation of what software should run on the PC side. I'm sure it's obvious to everyone who actually uses MIDI. Please remember I don't really use MIDI... I just write the low-level code for it. Need some step-by-step guidance about what I specifically need to do on the PC side. If it's only reproducible with Windows and I can't use Linux, also please remember I only ever use Windows for testing whether Teensy's software works on Windows. I'm not familiar at all with most Windows software, but I do have a Windows 10 test machine which I just re-imaged to a clean install for investigating 2 Windows-only issues yesterday.
 
Hi Paul. In my case (e.g. msg 46) the windows program I used to receive sysex dumps from the Teensy was "MidiOx" which is free but runs only on Windows. It's easy to use to assess this issue though Connect the Teensy, open the program and check your Teensy is shown in the options/midi devices tab, and select it as an input. Click on View/Sysex which produces two windows - a command, and display window. Right click on the display window, and select manual dump. The program is now waiting for the sysex dump from the Teensy, and will display and ongoing count of the number of bytes received until completed. Run this multiple times and check the byte count for missing data.

Note that the solution for T3.6 and 4.1 was different. And since the T3.6 is pretty much unavailable, it would be a better use of your time to focus on the T4.1 usb fix described by Rolf. Having migrated my T3.6 code to T4.1 now, I do find it a pain to have to remember to add the flush command after any midi send command to make the modified usb code work.

p.s. In the reverse direction, dumping sysex from MidiOx to teensy (T3.6 or 4.1), I do have one remaining issue I haven't solved yet. If I select the Teensy as both an input and output device in MidiOx, and try to do a sysex dump from MidiOx, sometimes the sysex is corrupted. If I disable the MidiOx input from the Teensy, the sysex is 100% reliable. Not sure if this is at all related, but thought I would mention it.
 
Hello Paul. Thank you for your time to solve the Midi reception problem in Teensy.


I have similar problem receiving Midi SysEx data on a Teensy 4.1.
I am using current Win10. For the USB transmission of SysEx data I use MIDI-OX, Bome SendSX and MIOS Studio.


My SysEx data block consists of 401 bytes. All apps have transmission errors because the Midi buffer in the Teensyduino Core Library is too small. I changed the buffer value for my synthesizer project ( https://forum.pjrc.com/threads/63255-New-Teensy-4-1-DIY-Synthesizer?p=326038&viewfull=1#post326038 ) to 512 bytes and everything works very well.


Attached is a SysEx file with a size of 401 bytes

teensy4-usb-midi-h.png



Bome SendSX

SysEx.png


Link: https://www.bome.com/products/sendsx#downloads

Great Thanks for your help :) Rolf from germany
 
Bringing up other MIDI issues on this thread is only going to slow me down, and may ultimately hinder a final solution to this issue of transmitting long sysex from Teensy to PC. Please, DO NOT make this more complicated.

If you can, please wait just a few days, to give me a little time to focus only on this specific issue. If other MIDI issues might be related (presumably the reason for bringing them up here), just be patient for a few days and if they are related and whatever is ultimately done about this large sysex transmit issue ends up solving those, then no need to start new threads. But if they aren't solved here, start a new thread. Don't make this thread more complicated. That's only going to end up being counterproductive.
 
I programmed a Teensy 4.1 with the code from msg #30 and plugged it into a Windows 10 machine, then ran MIDIOX.

This is the result when I press a pushbutton connected to Teensy.

capture.jpg

I scrolled up and it seems to be the exact same 130 byte message repeated many times. I can't see any which differ. Is this the expected result? What is the error?

MIDIOX doesn't seem to let me highlight the data and copy to clipboard. Is there any way to export?
 
I deleted "delay(delayAmt);" from the test program.

MIDIOX was complaining about lack of memory when Teensy reconnected. I rebooted the Windows machine and tried again. This time I got something that looks wrong. Is this similar to the results you're seeing?

capture2.jpg
 
MIDI-OX is not stable under Win10 (its an very old midi tool).
Bome SendSX is easier and better than Midi Ox.

Receive and copy an SysEx File in MIDI-OX:
View -> SysEx -> SysEx -> Receive Manual Dump

then press the Teensy button to send SysEx, wait a second, and then press Done. Now you can copy sysex to save.



SysEx 1.jpg
 
Last edited:
Rolfdegen said:
MIDI-OX is not stable under Win10 (its an very old midi tool).
Bome SendSX is easier and better than Midi Ox.
Click to expand...

Ran more tests with MIDI-OX while watching the USB packets with Total Phase Beagle 480 logged with a separate Linux machine. Will try SendSX late tonight or tomorrow. For now, I'm out of time for testing and might not even get this message fully written, but wanted to at least quickly share the results.

First, I can't reproduce the problem at all with the code of msg #30 as-is. But I can get corrupted receive in MIDI-OX if I delete the 1ms delay, causing Teensy 4.1 to transmit as fast as possible.

Everything I can see in the USB packets shows Teensy really did transmit all the sysex data correctly.

This is the wrong info I saw in MIDI-OX on my last test. It heard only 73 bytes in the last receive, and it's obviously fragments of data from 2 of the messages Teensy tried to send.

capture3.png

Here are the last 2 USB packets Teensy transmitted. The USB packets are all 512 bytes, except for the last one. Multiple sysex messages get packed into the same packet. I've highlighted the actual sysex data bytes.

packet1.jpg

packet2.jpg

As nearly as I can tell, Teensy is properly transmitting all the data its high-level code said to send. Something is going wrong on the PC side. Maybe something about the software just can't handle data coming in this quickly?


Rolfdegen said:
To test. Here is a 401 byte long SysEx file and a complete SoundBank with 128 SysEx files.
Click to expand...

Please explain step-by-step how I should use these files to test. Remember I have *NEVER* used Bome SendSX before, and I don't even use Windows much.

If you are again asking me to test something else, like PC transmitting sysex to Teensy, please stop now. Read msg #55. Don't make this thread more complicated by trying to get me to look at more than 1 issue. I will investigate other issues later, but not right now and absolutely not as part of this thread.
 
A few more quick tests with MIDI-OX. Deleting the usb_midi_flush_output() call from usb.c makes no difference. With the code from msg #30, I get missing date in MIDI-OX either with or without usb_midi_flush_output(), but only when I remove the 1ms delay. If I put the delay in, I always get correct results in MIDI-OX, either with or without usb_midi_flush_output() in usb.c.
 
Tried Bome SendSX briefly. It does much better than MIDI-OX. But ultimately looks like the same problem. Teensy definitely is transmitting all the sysex messages in the USB packets. Every packet does get an ACK by the PC's USB host controller. But not all the data gets received and shown in SendSX.

Again, I can't see any difference by having or deleting usb_midi_flush_output() in usb.c.
 
Everything I'm seeing so far looks like Teensy is transmitting correct and the problem is software on the Windows side can't keep up and handles that scenario quite badly.

I really don't see any possible solution other than adding code to throttle the output rate.
 
I can't remember exactly what I was doing in Arduino core library for send SysEx Datas to my PC :confused:

But.. I changed the midi buffer in Arduino core/teensy4/usb_midi.h form 290 to 512 Byte for receive SysEx Data from PC.


I can transfer 401 byte blocks from Teensy 4.1 to my PC without errors ( usbMIDI.sendSysEx(lenght, buffer, true); )

System Info:
Windows 10 Home 22H2 Core i5 4.1GHz 32GB Ram
Arduino Version 1.6/1.8 Teensy 4.1
Microchip Studio 7 with vMicro


Bome.PNGMicrochip.jpg
 
Last edited:
@rvh - Any thoughts? I can't see any change with vs without usb_midi_flush_output(). Really looks like a "simple" issue of Windows badly handling rapid arrival of too much data. Should I run more tests? What should I do to see a difference by removing the call to usb_midi_flush_output() ?
 
Rolfdegen said:
My SysEx data block consists of 401 bytes. All apps have transmission errors because the Midi buffer in the Teensyduino Core Library is too small.
Click to expand...

See the 3-input System Exclusive callback in File > Examples > Teensy > USB_MIDI > InputFunctionsComplete.

https://github.com/PaulStoffregen/T...tionsComplete/InputFunctionsComplete.ino#L120

This was added to the core library USB MIDI to allow receiving sysex larger than the buffer. But it was only ever documented in the examples (InputRead mentions it too). But it wasn't ever properly documented on the USB MIDI page. I really should add something about it there, even if just a mention of the example code.
 
Paul. Thank you for your effort. I have to go to sleep now. Tomorrow I have time to program again. Thanks very much :)
 
Hi Paul, did you add an explicit flush command after the usb_send when commenting out the flush command in usb.c? See message #21. If not, commenting out the flush command in usb.c on its own won't fix the issue.

Although altering usb.c (and adding a manual flush after the send) fixed the T4.1 sysex problem for me when sending large sysex files, it's a bit of a pain because I now have to add the explicit flush whenever I do any kind of usb midi send, not just sysex dumps.

Also, in my sysex dumps a patch contains 6 blocks of 212 bytes that are delayed by 5ms to slow things down for Windows/MidiOx. But even then, if I don't make the usb.c changes, I get intermittent sysex data corruption. However, to see the errors I have to send lots of patches, rather than just a few. My tests were typically done with a bank of 100 patches.
 
Last edited:
Also, even if it is a "Windows behaving badly" issue it would be highly desirable to find a workaround in the Teensy library code for USB sysex sends. A key use of the sysex-send to the computer is to back up patch data, so it must be 100% reliable. Sacrificing usb transmission speed (for any required delays) in the case of sysex usually wouldn't be an issue, but it would be good to avoid impacting any non-sysex usb midi send command if possible.
 
Yesterday I tested only with code from msg #30, without the delay, and experimented with different delays using MIDI-OX and Bome SendSX. I didn't put usb_midi_flush_output() into that program.

rvh said:
Hi Paul, did you add an explicit flush command after the usb_send when commenting out the flush command in usb.c? See message #21.
Click to expand...

Tried this just now (no delay and usb_midi_flush_output() after sending each sysex), and with usb_midi_flush_output() deleted from usb.c. I still see missing data with both MIDI-OX and SendSX. The problem is not as frequent, but still definitely exists.

Looking at the actual USB packets, Teensy is sending 176 byte USB packets. Normally it packs the outgoing data efficiently into 512 byte packets. I haven't analyzed to timing carefully, but a quick look judging by the mix of bulk data packets and SOF packets seems to show usually 1 or sometimes 2 packets per 125us USB micro-frame. The number of packets per micro-frame looks pretty similar in both tests.

Adding usb_midi_flush_output() has a side effect of causing the sustained data rate to drop to only about 1/3rd of the speed.

My best guess is you're probably seeing data received properly because of the side effect of slowing the transmission. Your Windows machine is probably faster than my test machine, which is an 11 year old laptop with Intel i7-2760 processor running at 2.4 GHz. My machine with either MIDI-OX or SendSX often does receive all the messages. But if I keep pressing the pushbutton rapidly (msg #30 code), occasionally I do see both programs sometimes report receiving lengths other than 130.

When Teensy sends maximum speed with messages efficiently packed into 512 byte packets, for reasons I don't understand the Windows driver still seems to transfer about the same number of packets per second (or per 125us microframe).

It's always 1 or sometimes 2 packets per microframe, so larger packets means more total data transfer. FWIW, that 1-2 is nowhere near the 13 max size packets 480 Mbit/sec USB can theoretically transfer in each 125us if both host and device operate at the maximum possible speed. It's easy to see in the USB capture that this 1-2 packets per 125us is coming from the host side, because there are almost no NAK response from Teensy where the host controller tried to read more but Teensy didn't have a packet to transmit.
 
Here is the code I tested just now, with usb_midi_flush_output() deleted from usb.c. My i7-2760 laptop with Windows 10 can often receive all the sysex messages correctly, but repeatedly pressing the pushbutton still shows occasional errors.

Code: 
#include <Bounce2.h>

#define chunkSize 130
#define Blocknumbers 128
#define delayAmt 1
#define buttonPin 2

const int ledPin = 13;
Bounce  myButton  = Bounce();
boolean startFlag = false;
byte data[chunkSize];


void fillBuff() {
  // set Start and End Byte for SysEx
  data[0] = 0xF0;
  data[129] = 0xF7;
  // fill with data
  for (uint16_t ct = 1; ct < (chunkSize - 1); ct++) {
    int scratch = (ct & 0b01111111);
    data[ct] = scratch;
  }
}

void sendSysExData () {
  usbMIDI.sendSysEx(chunkSize, data, true);
  digitalWrite(ledPin, HIGH);
}

void setup() {
  usbMIDI.begin();
  pinMode(ledPin, OUTPUT);
  pinMode(buttonPin, INPUT_PULLUP);
  myButton.attach(buttonPin);
  myButton.interval(5); // 5ms for Keys debouncing
  fillBuff();
}

void loop() {
  // press Key for transmit SysEx data
  myButton.update();
  if (myButton.fallingEdge()) {
    startFlag = true;
  }

  // send 128 Blocks with 130 Byte
  if (startFlag == true) {
    for (uint8_t i = 0; i < Blocknumbers; i++) {
      sendSysExData();
      usb_midi_flush_output(); // see msg #21 and #69
      //delay(delayAmt);
      //delayMicroseconds(195); // minimum for Bome SendSx
      //delayMicroseconds(260); // minimum for MIDI-OX
    }
    startFlag = false;
    digitalWrite(ledPin, LOW);
  }
  
  while (usbMIDI.read()) {
    // ignore incoming messages
  }
}
 
Tried the test program from msg #46. Indeed it has problems on Teensy 3.6 when sending to MIDI-OX, because MIDI-OX echos messages and that code isn't reading the input to discard incoming MIDI, so the USB buffer memory fills up. It works fine with this change.

Code: 
#include <MIDI.h>
MIDI_CREATE_INSTANCE(HardwareSerial, Serial1, MIDI );  // not used

void setup() {
  pinMode(2, INPUT_PULLUP);
  MIDI.begin(MIDI_CHANNEL_OMNI); // midi serial init
}

void loop() {
  if (!digitalRead(2)) {
    byte patchData[50] = {0};
    patchData[0] = 0xF0;
    patchData[49] = 0xF7;

    for (int P = 0; P < 50; P++) {
      for (int i = 0; i < 20; i++) {
        usbMIDI.sendSysEx( 50, patchData, true);
        ignore();
        delay(10);
      }
      delay(50);
    }
  }
  ignore();
}

void ignore() {
  // MIDI Controllers should discard incoming MIDI messages.
  // http://forum.pjrc.com/threads/24179-Teensy-3-Ableton-Analog-CC-causes-midi-crash
  while (usbMIDI.read()) {
    // ignore incoming messages
  }
}
 
Looks like I spoke too soon about msg #46 code. I was looking for truncated or corrupted messages. Didn't see any, but with MIDI-OX receiving Teensy can't send all the messages. Many of them are simply missing. SendSX works fine because it doesn't echo.

Ignoring the incoming messages MIDI-OX echoes isn't so simple. It can't be done just once after after transmitting. To keep the buffers from getting messed up, it needs to be done during the delays, like this:

Code: 
void setup() {
  pinMode(2, INPUT_PULLUP);
  delay(1); // allow time for pullup to bring pin 2 high
}

void delay_while_discarding_input(unsigned int msec) {
  elapsedMillis t = 0;
  while (t < msec) {
    usbMIDI.read(); // read and ignore incoming USB MIDI
  }
}

void loop() {
  if (!digitalRead(2)) {
    byte patchData[50] = {0};
    patchData[0] = 0xF0;
    patchData[49] = 0xF7;

    for (int P = 0; P < 50; P++) {
      for (int i = 0; i < 20; i++) {
        patchData[3] = i;
        patchData[4] = P;
        usbMIDI.sendSysEx( 50, patchData, true);
        delay_while_discarding_input(10);
      }
      delay_while_discarding_input(50);
    }
  }
  usbMIDI.read();
}

Teensy 4.x doesn't need this, because the transmit and receive buffers aren't shared. It's really only an issue with Teensy 3.x.
 
rvh said:
Also, even if it is a "Windows behaving badly" issue it would be highly desirable to find a workaround in the Teensy library code for USB sysex sends. A key use of the sysex-send to the computer is to back up patch data, so it must be 100% reliable.
Click to expand...

I'm sad to say there is no easy answer to how much we need to throttle Teensy 4.x transmit speed. Windows is sending an IN token to tell Teensy it want more data. Then it is sending an ACK token after Teensy transmits the data, to indicate successful reception.

Whether intentional throttling belongs inside the low-level MIDI code or high-level application (Arduino sketch) is a good question.

Unfortunately, the only way to make this 100% reliable is to pace your transmitting. For sysex messages of 384 bytes or smaller, at 480 Mbit/sec speed, looks like sending about 4 per millisecond is pushing the limit. But this isn't any sort of confirmed limit, only my experimental result with MIDI-OX and SendSX running on a rather clean Windows 10 install (almost no other software installed, nothing else running).
 
You must log in or register to reply here.
Back
Top