Hi,
in addition to data channel 0 of sai1, I got the data channels 1/2/3 working. So it's possible to receive 2/4/6 of 8 audio channels with sai1 of T4. The problem is that the block that implemented is not 100% compatible with the teensy audio library, since I am using a 32 bit floating point/ 48kHz Version of the lib. Also, I configured the input as I2s slave and resample all data to 48kHz. Nevertheless my block also implements a 'begin', 'isr' and 'config_i2s' function, which basically should also work with the original audio library or at least can serve as a starting point if somebody needs to impement e.g. a quad channel i2s input for the teensy 4.
So here are the code snippets for the functions mentioned above:

begin():
Code:
void AudioInputAsyncI2sOct::begin(void)
{
	dma.begin(true); // Allocate the DMA channel first
	CORE_PIN8_CONFIG  = 3;  //1:RX_DATA0					(IOMUXC_SW_MUX_CTL_PAD_GPIO_B1_00)
	CORE_PIN32_CONFIG  = 3; //1:RX_DATA3 (SAI1_TX_DATA01)	(IOMUXC_SW_MUX_CTL_PAD_GPIO_B0_12)
	CORE_PIN9_CONFIG  = 3;  //1:RX_DATA2 (SAI1_TX_DATA02)	(IOMUXC_SW_MUX_CTL_PAD_GPIO_B0_11)
	CORE_PIN6_CONFIG  = 3;  //1:RX_DATA1 (SAI1_TX_DATA03)	(IOMUXC_SW_MUX_CTL_PAD_GPIO_B0_10)
	CORE_PIN7_CONFIG  = 3;  // (SAI1_TX_DATA00)				(IOMUXC_SW_MUX_CTL_PAD_GPIO_B1_01)
	IOMUXC_SAI1_RX_DATA0_SELECT_INPUT = 2;
	IOMUXC_SAI1_RX_DATA1_SELECT_INPUT = 1;//	(IMXRT_IOMUXC_b.offset198)
	IOMUXC_SAI1_RX_DATA2_SELECT_INPUT = 1;//	(IMXRT_IOMUXC_b.offset19C)
	IOMUXC_SAI1_RX_DATA3_SELECT_INPUT = 1;//	(IMXRT_IOMUXC_b.offset1A0)
	config_i2s();

	const uint32_t noByteMinorLoop=4*NO_CHANNELS/2;	//at each minor one sample of each data channel is read (e.g. rx0 channel0, rx1 channel0, rx2 channel0, rx3 channel0 )
	dma.TCD->SADDR = (void *)((uint32_t)&I2S1_RDR0);
	dma.TCD->SOFF = 4;
	dma.TCD->ATTR = DMA_TCD_ATTR_SSIZE(2) | DMA_TCD_ATTR_DSIZE(2);
	dma.TCD->NBYTES_MLNO =  DMA_TCD_NBYTES_MLOFFYES_NBYTES(noByteMinorLoop) | DMA_TCD_NBYTES_SMLOE | 
	 					DMA_TCD_NBYTES_MLOFFYES_MLOFF(-noByteMinorLoop);
	dma.TCD->SLAST = -noByteMinorLoop;
	dma.TCD->DADDR = i2s_rx_buffer;
	dma.TCD->DOFF = 4;
	dma.TCD->CITER_ELINKNO = sizeof(i2s_rx_buffer) / noByteMinorLoop;
	dma.TCD->DLASTSGA = -sizeof(i2s_rx_buffer);
	dma.TCD->BITER_ELINKNO = sizeof(i2s_rx_buffer) / noByteMinorLoop;
	dma.TCD->CSR = DMA_TCD_CSR_INTHALF | DMA_TCD_CSR_INTMAJOR;
	dma.triggerAtHardwareEvent(DMAMUX_SOURCE_SAI1_RX);
	dma.enable();

	I2S1_RCSR=0;
	I2S1_RCSR = I2S_RCSR_RE | I2S_RCSR_BCE | I2S_RCSR_FRDE | I2S_RCSR_FR;
	//I2S1_TCSR = I2S_TCSR_TE | I2S_TCSR_BCE | I2S_TCSR_FRDE | I2S_TCSR_FR;

	dma.attachInterrupt(isr);
#ifdef DEBUG_ASYNC_I2S_IN
	while (!Serial);
#endif
	for (uint8_t i =0; i< NO_CHANNELS; i++){
		memset(buffer[i], 0, bufferLength*sizeof(float));
	}
}
config_i2s():
Code:
void AudioInputAsyncI2sOct::config_i2s(){
	if (I2S1_TCSR & I2S_TCSR_TE){
		I2S1_TCSR &=~I2S_TCSR_TE;
	}
	if (I2S1_RCSR & I2S_RCSR_RE){
		I2S1_RCSR &=~I2S_RCSR_RE;
	}

	CCM_CCGR5 |= CCM_CCGR5_SAI1(CCM_CCGR_ON);
	CORE_PIN21_CONFIG = 3;  //1:RX_BCLK
	CORE_PIN20_CONFIG = 3;  //1:RX_SYNC
	IOMUXC_SAI1_RX_BCLK_SELECT_INPUT = 1; // 1=GPIO_AD_B1_11_ALT3, page 868
	IOMUXC_SAI1_RX_SYNC_SELECT_INPUT = 1; // 1=GPIO_AD_B1_10_ALT3, page 872

	// configure transmitter
	I2S1_TMR = 0;
	I2S1_TCR1 = I2S_TCR1_RFW(1);  // watermark at half fifo size
	I2S1_TCR2 = I2S_TCR2_SYNC(1)	//1...transmitter is running synchronously with receiver, 0.. running asynchronously
				| I2S_TCR2_BCP;		//Bit clock polarity: active low Todo: check
	I2S1_TCR3 = 0;	// disable all transmit channels for now, I2S_TCR3_TCE;		//Transmit channel enable
	I2S1_TCR4 = I2S_TCR4_FRSZ(1)	//two words per frame
				| I2S_TCR4_SYWD(31)	//sync width: 31 -> 32bit = 1 word
				| I2S_TCR4_MF		//most significant bit first
				| I2S_TCR4_FSE		//Frame sync asserts one bit before the first bit of the frame
				| I2S_TCR4_FSP;		//frame sync polarity: frame sync is active low
	I2S1_TCR5 = I2S_TCR5_WNW(31)	// word N width
				| I2S_TCR5_W0W(31)	// word 0 width
				| I2S_TCR5_FBT(31);	//first bit shifted

	// configure receiver
	I2S1_RMR = 0;
	I2S1_RCR1 = I2S_RCR1_RFW(1);
	I2S1_RCR2 = I2S_RCR2_SYNC(0)	//1...receiver is running synchronously with transmitter, 0.. running asynchronously
				| I2S_TCR2_BCP;	//Bit Clock is active low with drive outputs on falling edge and sample inputs on rising edge
	uint32_t enableChannel=I2S_RCR3_RCE;
	I2S1_RCR3=0;
	for (uint8_t i =0; i < NO_CHANNELS/2; i++){
		I2S1_RCR3 |= enableChannel;
		enableChannel=enableChannel<<1;
	}
	I2S1_RCR4 = I2S_RCR4_FRSZ(1)
				| I2S_RCR4_SYWD(31)
				| I2S_RCR4_MF
				| I2S_RCR4_FSE
				| I2S_RCR4_FSP;
	I2S1_RCR5 = I2S_RCR5_WNW(31)
				| I2S_RCR5_W0W(31)
				| I2S_RCR5_FBT(31);
}
isr():
Code:
void AudioInputAsyncI2sOct::isr(void)
{
	const int32_t *src, *end;
	uint32_t daddr = (uint32_t)(dma.TCD->DADDR);
	dma.clearInterrupt();

	if (daddr < (uint32_t)i2s_rx_buffer + sizeof(i2s_rx_buffer) / 2) {
		// DMA is receiving to the first half of the buffer
		// need to remove data from the second half
		src = (int32_t *)&i2s_rx_buffer[ASYNC_I2S_RX_BUFFER_LENGTH/2];
		end = (int32_t *)&i2s_rx_buffer[ASYNC_I2S_RX_BUFFER_LENGTH];
		//if (AudioInputAsyncI2sOct::update_responsibility) AudioStream::update_all();
	} else {
		// DMA is receiving to the second half of the buffer
		// need to remove data from the first half
		src = (int32_t *)&i2s_rx_buffer[0];
		end = (int32_t *)&i2s_rx_buffer[ASYNC_I2S_RX_BUFFER_LENGTH/2];
	}
	if (buffer_offset >=resample_offset ||
		(buffer_offset + noSamplerPerIsr) < resample_offset) {
		#if IMXRT_CACHE_ENABLED >=1
		arm_dcache_delete((void*)src, sizeof(i2s_rx_buffer) / 2);
		#endif
		float *destEvenChannels[NO_CHANNELS/2];
		float *destOddChannels[NO_CHANNELS/2];
		for (uint8_t i =0; i< NO_CHANNELS/2; i++){
			destEvenChannels[i]=&(buffer[2*i][buffer_offset]);
			destOddChannels[i]=&(buffer[2*i+1][buffer_offset]);
		}
		const float factor= pow(2., 31.);
		do {		
			for (uint8_t i =0; i< NO_CHANNELS/2; i++)	{
				*destEvenChannels[i]++ = (float)(*src)/factor;
				++src;
			}		
			for (uint8_t i =0; i< NO_CHANNELS/2; i++)	{
				*destOddChannels[i]++ = (float)(*src)/factor;
				++src;
			}
		} while (src < end);
		buffer_offset=(buffer_offset+noSamplerPerIsr)%bufferLength;
	}
#ifdef DEBUG_ASYNC_I2S_IN
	else {
		bufferOverflow=true;
	}
#endif	
}
The code won't work out of the box, since a few variables are not defined (e.g. NO_CHANNELS should be 2, 4, 6 or 8. 'noSamplerPerIsr' are the number of samples that are tranfered per isr call.), but I hope that the Code snippets are still helpful.

Best regards
Alex