diff --git a/src/gui/event-editor/gcal-schedule-section.c b/src/gui/event-editor/gcal-schedule-section.c
index 17eed558..ab9c1668 100644
--- a/src/gui/event-editor/gcal-schedule-section.c
+++ b/src/gui/event-editor/gcal-schedule-section.c
@@ -119,6 +119,8 @@ gcal_schedule_values_copy (const GcalScheduleValues *values)
   GcalScheduleValues *copy = g_new0 (GcalScheduleValues, 1);
 
   copy->all_day = values->all_day;
+  copy->orig_date_start = values->orig_date_start ? g_date_time_ref (values->orig_date_start) : NULL;
+  copy->orig_date_end = values->orig_date_end ? g_date_time_ref (values->orig_date_end) : NULL;
   copy->date_start = values->date_start ? g_date_time_ref (values->date_start) : NULL;
   copy->date_end = values->date_end ? g_date_time_ref (values->date_end) : NULL;
   copy->recur = values->recur ? gcal_recurrence_ref (values->recur) : NULL;
@@ -889,8 +891,12 @@ gcal_schedule_values_from_event (GcalEvent      *event,
       GcalRecurrence *recur = gcal_event_get_recurrence (event);
 
       values->all_day = gcal_event_get_all_day (event);
-      values->date_start = g_date_time_ref (gcal_event_get_date_start (event));
-      values->date_end = g_date_time_ref (gcal_event_get_date_end (event));
+
+      values->orig_date_start = g_date_time_ref (gcal_event_get_date_start (event));
+      values->date_start = g_date_time_ref (values->orig_date_start);
+
+      values->orig_date_end = g_date_time_ref (gcal_event_get_date_end (event));
+      values->date_end = g_date_time_ref (values->orig_date_end);
 
       if (recur)
         {
@@ -932,6 +938,8 @@ test_turning_all_day_on_displays_sensible_dates (void)
 
   *values = (GcalScheduleValues) {
     .all_day = FALSE,
+    .orig_date_start = g_date_time_ref (date),
+    .orig_date_end = g_date_time_ref (one_hour_later),
     .date_start = g_date_time_ref (date),
     .date_end = g_date_time_ref (one_hour_later),
     .recur = NULL,
diff --git a/src/gui/event-editor/gcal-schedule-section.h b/src/gui/event-editor/gcal-schedule-section.h
index 35ec0a46..fda4cedd 100644
--- a/src/gui/event-editor/gcal-schedule-section.h
+++ b/src/gui/event-editor/gcal-schedule-section.h
@@ -40,8 +40,16 @@ G_DECLARE_FINAL_TYPE (GcalScheduleSection, gcal_schedule_section, GCAL, SCHEDULE
 typedef struct
 {
   gboolean all_day;
+
+  /* Original times from the GcalEvent.  We keep these around to be able to reconstruct
+   * the event's duration in case the all-day toggle gets turned on and off repeatedly.
+   */
+  GDateTime *orig_date_start;
+  GDateTime *orig_date_end;
+
   GDateTime *date_start;
   GDateTime *date_end;
+
   GcalRecurrence *recur;
   GcalTimeFormat time_format;
 } GcalScheduleValues;