Thank you both!

@WMXZ, your code returns wrong numbers for certain values. Some lead to slightly off frequencies.

@Theremingenieur, your idea works - I wrote some code for it:

Code:

void I2S_dividers2( int fsamp, int nbits, int TCR_DIV2)
{ // nbits is number of bits / frame
const uint8_t primes[] = {2, 3, 5, 7, 11, 13};
unsigned x = FCPU;
unsigned y = fsamp * nbits * (TCR_DIV2 + 1) * 2;
unsigned ip = 0;
unsigned p;
unsigned divi = 1;
unsigned mult = 1;
do {
p = primes[ip];
unsigned xx = 0;
while (x % p == 0) {
x /= p;
xx++;
}
unsigned yy = 0;
while (y % p == 0 ) {
y /= p;
yy++;
}
if (xx > 0 || yy > 0) {
int z = min(xx, yy);
divi *= powf(p, xx - z);
mult *= powf(p, yy - z);
// Serial.printf("L:%d^%d=%d R:%d^%d=%d\n", p,xx-z, (int)powf(p,xx-z), p, yy-z, (int)powf(p,yy-z) );
}
ip++;
} while ((x > 1 || y > 1) && (ip < sizeof(primes)));
Serial.printf("\nMULT: %d DIV: %d\n", mult, divi);
}

(call with I2S_dividers2(44100, 32, 3) for example)

Perhaps there are ways to optimize it more, but it is fast enough - ~0.15ms on a T3.2/98MHz

(edit: if someone looks for a very fast integer pow(): https://gist.github.com/orlp/3551590)