Electronic – Atmel SAMC21 MCU internal oscillator based counter inaccurate

I am programming a SAMC21 for a control system and I need a milliseconds counter similar to the millis function in arduino to keep track of samples, etc. To do this I first set up a clock channel for the internal oscillator (48MHz) whose output I divide by 48. I then set up an 8 bit Timer/Counter with a period of 100. Everytime the TC reaches 100 it overflows and I use a callback to call a function which simply increments my millis counter variable (volatile uint32_t). This works fine but the counter is quite slower than it should be and delays by between 5 and 15s every minute. Why is that so and how could I make it more accurate?

Note: I use the UART driver over the Debug USB to read the counter value.

Here is the code which is called appropriately in main:

 void configure_gclock_generator(void) // Configuration of internal oscillator and channel
 {
     struct system_gclk_gen_config gclock_gen_conf;
     system_gclk_gen_get_config_defaults(&gclock_gen_conf);
     gclock_gen_conf.source_clock    = SYSTEM_CLOCK_SOURCE_OSC48M;
     gclock_gen_conf.division_factor = 48;
     system_gclk_gen_set_config(GCLK_GENERATOR_1, &gclock_gen_conf);
     system_gclk_gen_enable(GCLK_GENERATOR_1);
 }
 void configure_gclock_channel(void)
 {
     struct system_gclk_chan_config gclk_chan_conf;
     system_gclk_chan_get_config_defaults(&gclk_chan_conf);
     gclk_chan_conf.source_generator = GCLK_GENERATOR_1;
     system_gclk_chan_set_config(TC3_GCLK_ID, &gclk_chan_conf);
     system_gclk_chan_enable(TC3_GCLK_ID);
 }

void configure_tc(void) // COnfiguration of TC on internal oscillator
{
    struct tc_config config_tc;
    tc_get_config_defaults(&config_tc);
    config_tc.counter_size = TC_COUNTER_SIZE_8BIT;
    config_tc.clock_source = GCLK_GENERATOR_1;
    //config_tc.clock_prescaler = TC_CLOCK_PRESCALER_DIV1;
    config_tc.counter_8_bit.period = 100;
    //config_tc.counter_16_bit.compare_capture_channel[0] = 48;
    tc_init(&tc_instance, CONF_TC_MODULE, &config_tc);
    tc_enable(&tc_instance);
}

void millis_count(struct tc_module *const module_inst) { //millis increment function
    millis++;
    if (millis>=1000000000) {
        millis=millis-start;
        start=0;
    }
}

void configure_tc_callbacks(void) //Set up of Callback for millis function.
{
    tc_register_callback(&tc_instance, millis_count,TC_CALLBACK_OVERFLOW);
    //tc_register_callback(&tc_instance, millis_count,TC_CALLBACK_CC_CHANNEL0);
    tc_enable_callback(&tc_instance, TC_CALLBACK_OVERFLOW);
    //tc_enable_callback(&tc_instance, TC_CALLBACK_CC_CHANNEL0);
}