Changing ADC clock speed does not change sampling speed [XMega]

I am trying to measure how fast my ADC samples arrive on an XMega128A4U. I have set up a timer and the ADC, and each time I have a result I let the DMA pull the CNT register from the timer and the result register from the ADC. After sampling 1024 samples, the DMA flags it is finished an I print the result out on USB.

My issue is that even though I change the ADC prescaler, the delta between each sample is the same.

System clock 32MHz Internal RC.

ADC Prescaler 16 (32MHz/16 = 2MHz)

TC0: 930 - CH0=1410
TC0: 931 - CH0=1410
TC0: 932 - CH0=1378
TC0: 934 - CH0=1366
TC0: 936 - CH0=1352
TC0: 937 - CH0=1342
TC0: 939 - CH0=1330
TC0: 940 - CH0=1319
TC0: 942 - CH0=1319
TC0: 944 - CH0=1297
TC0: 945 - CH0=1290

ADC prescaler 512 (32Mhz/512 = 62.5kHz)

TC0: 933 - CH0=997
TC0: 934 - CH0=994
TC0: 935 - CH0=998
TC0: 936 - CH0=986
TC0: 936 - CH0=1002
TC0: 937 - CH0=987
TC0: 938 - CH0=985
TC0: 939 - CH0=995
TC0: 940 - CH0=987
TC0: 941 - CH0=987

The timer is running at 4MHz (32Mhz/8) so the time between each tick is 250nS. This just doesn't seem right! What am I missing here? Is there something else I need to set other than the prescaler?

Full code:

#define F_CPU 32000000UL
#include <asf.h>
#include <util\delay.h>

#define OVERSAMPLING_FACTOR 10
#define DMA_BUFFER_SIZE     (1 << OVERSAMPLING_FACTOR)
#define DMA_ADC_CH0         0
#define DMA_ADC_CH1         1
#define DMA_ADC_CH2         2

static uint16_t dma_buffer[DMA_BUFFER_SIZE];
static uint16_t tc_buffer[DMA_BUFFER_SIZE];

volatile bool dma_is_full = false;

void dma_init(void);
void adc_init(void);
void timer_init(void);

static void dma_adc_res_done(enum dma_channel_status status)
{
    if (status == DMA_CH_TRANSFER_COMPLETED)
    {
        dma_is_full = true;
    }   
}


int main (void)
{
    irq_initialize_vectors();
    cpu_irq_enable();

    sysclk_init();
    board_init();
    dma_init();
    adc_init();
    timer_init();

    stdio_usb_init();
    stdio_usb_enable();

    // Power up target board for current measurement
    ioport_set_pin_high(CURRENT_2);

    // Start ADC and timer
    tc_enable(&TCC0);
    ADCA.CTRLA |= (ADC_ENABLE_bm << ADC_ENABLE_bp);
    while (true) 
    {
        sleepmgr_enter_sleep();

        // Sleeping while DMA not finished
        if(dma_is_full == true)
        {
            _delay_ms(500); // Wait so USB has time to connect before printing
            if (udi_cdc_is_tx_ready())
            {
                char line_buffer[20];
                uint8_t line_length;

                for (uint16_t i = 0; i < DMA_BUFFER_SIZE; i++)
                {
                    line_length = sprintf(  line_buffer,
                                            "TC0: %u - CH0=%u\r\n",
                                            tc_buffer[i],
                                            dma_buffer[i]
                                            );

                    udi_cdc_write_buf(line_buffer, line_length);
                }
            }
            dma_is_full = false;
        }
        // Enabling channels again to start next transfer
//          dma_channel_enable(DMA_ADC_CH0);
//          dma_channel_enable(DMA_ADC_CH2);

    }

}


void dma_init(void)
{
    // =================================================
    // Config DMA Channel 0 for ADC samples
    // =================================================    

    struct dma_channel_config dmach_adc_res1_conf;
    memset(&dmach_adc_res1_conf, 0, sizeof(dmach_adc_res1_conf));

    dma_channel_set_burst_length(&dmach_adc_res1_conf, DMA_CH_BURSTLEN_2BYTE_gc);   
    dma_channel_set_transfer_count(&dmach_adc_res1_conf, DMA_BUFFER_SIZE*2);

    dma_channel_set_src_reload_mode(&dmach_adc_res1_conf, DMA_CH_SRCRELOAD_BURST_gc);
    dma_channel_set_dest_reload_mode(&dmach_adc_res1_conf, DMA_CH_DESTRELOAD_BLOCK_gc);

    dma_channel_set_src_dir_mode(&dmach_adc_res1_conf, DMA_CH_SRCDIR_INC_gc);
    dma_channel_set_dest_dir_mode(&dmach_adc_res1_conf, DMA_CH_DESTDIR_INC_gc);

    dma_channel_set_single_shot(&dmach_adc_res1_conf);
    dma_channel_set_source_address(&dmach_adc_res1_conf, (uint16_t)(uintptr_t)&ADCA.CH0RES);
    dma_channel_set_destination_address(&dmach_adc_res1_conf, (uint16_t)(uintptr_t)dma_buffer);

    // Set DMA trigger source to CH2 complete interrupt flag
    dma_channel_set_trigger_source(&dmach_adc_res1_conf, DMA_CH_TRIGSRC_ADCA_CH0_gc);   

    // =================================================
    // Config DMA Channel 2 to pull timestamps from TC0
    // =================================================
    struct dma_channel_config dmach_tc_conf;
    memset(&dmach_tc_conf, 0, sizeof(dmach_tc_conf));

    dma_channel_set_burst_length(&dmach_tc_conf, DMA_CH_BURSTLEN_2BYTE_gc);
    dma_channel_set_transfer_count(&dmach_tc_conf, DMA_BUFFER_SIZE*2);

    dma_channel_set_src_reload_mode(&dmach_tc_conf, DMA_CH_SRCRELOAD_BURST_gc);
    dma_channel_set_dest_reload_mode(&dmach_tc_conf, DMA_CH_DESTRELOAD_BLOCK_gc);

    dma_channel_set_src_dir_mode(&dmach_tc_conf, DMA_CH_SRCDIR_INC_gc);
    dma_channel_set_dest_dir_mode(&dmach_tc_conf, DMA_CH_DESTDIR_INC_gc);

    dma_channel_set_single_shot(&dmach_adc_res1_conf);
    dma_channel_set_source_address(&dmach_tc_conf, (uint16_t)(uintptr_t)&TCC0.CNT);
    dma_channel_set_destination_address(&dmach_tc_conf, (uint16_t)(uintptr_t)tc_buffer);
    dma_channel_set_trigger_source(&dmach_tc_conf, DMA_CH_TRIGSRC_ADCA_CH0_gc);


    // Enable DMA and configure channels
    dma_enable();
    dma_set_callback(DMA_ADC_CH0, dma_adc_res_done);
    dma_channel_set_interrupt_level(&dmach_adc_res1_conf, DMA_INT_LVL_MED);
    dma_channel_write_config(DMA_ADC_CH0, &dmach_adc_res1_conf);

    dma_channel_set_interrupt_level(&dmach_tc_conf, DMA_INT_LVL_MED);
    dma_channel_write_config(DMA_ADC_CH2, &dmach_tc_conf);

    dma_channel_enable(DMA_ADC_CH0);
    dma_channel_enable(DMA_ADC_CH2);


}


void adc_init(void)
{
    PR.PRPA &= ~PR_ADC_bm;      // Power up
    ADCA.CTRLA = ADC_FLUSH_bm;  // Cancel pending conversions (in case it was already powered)
    ADCA.CTRLA = 0;             // Disable ADC
    ADCA.EVCTRL = 0;                

    /*  Set up ADCA

        Full speed, no limit (2 MSPMS)
        12 bit resolution
        Unsigned conversion
        ADC Reference AREFB (2.5V)
        Freerunning mode
    */
    ADCA.CTRLB  |= ADC_CURRLIMIT_NO_gc 
                | ADC_RESOLUTION_12BIT_gc 
                | ADC_SIGN_OFF 
                | ADC_FREERUN_bm
                ;   

    ADCA.CTRLB &= ~ADC_CONMODE_bm;  
    ADCA.REFCTRL |= ADC_REF_AREFB;
    ADCA.PRESCALER = ADC_PRESCALER_DIV512_gc; // 32MHz/16 = 2MHz = ADC max clock speed


    /*  Set up CH0 on ADCA

        Single-ended measurement on VOUT3
        1x gain
        Interrupt flag set on conversion complete (used by DMA)
        Interrupt OFF (DMA polls interrupt flag, doesn't need ADC ISR)
    */
    //! Note: sys_clk = 32000000
    ADCA.CH0.CTRL |= ADC_CH_INPUTMODE_SINGLEENDED_gc | ADC_CH_GAIN_1X_gc;
    ADCA.CH0.MUXCTRL = VOUT_3;
    ADCA.CH0.INTFLAGS = 0xFF;       // Clear
    ADCA.CH0.INTCTRL |= ADC_CH_INTMODE_COMPLETE_gc | ADC_CH_INTLVL_OFF_gc;  
}

void timer_init(void)
{
    tc_enable(&TCC0);
    tc_set_wgm(&TCC0, TC_WG_NORMAL);
    tc_write_clock_source(&TCC0, TC_CLKSEL_DIV8_gc); // 4MHz
    tc_disable(&TCC0);
}