diff --git a/data/ui/source-dialog.ui b/data/ui/source-dialog.ui
index 68504b03..f8a52dbf 100644
--- a/data/ui/source-dialog.ui
+++ b/data/ui/source-dialog.ui
@@ -40,6 +40,7 @@
             <property name="can_focus">False</property>
             <property name="hexpand">True</property>
             <property name="vexpand">True</property>
+            <property name="vhomogeneous">False</property>
             <child>
               <object class="GtkGrid" id="edit_grid">
                 <property name="visible">True</property>
@@ -259,7 +260,7 @@
         <child>
           <object class="GtkButton" id="cancel_button">
             <property name="label" translatable="yes">Cancel</property>
-            <property name="visible">True</property>
+            <property name="visible" bind-source="headerbar" bind-property="show-close-button" bind-flags="default|sync-create|invert-boolean"/>
             <property name="can_focus">True</property>
             <property name="receives_default">True</property>
             <signal name="clicked" handler="action_widget_activated" object="GcalSourceDialog" swapped="no"/>
@@ -268,7 +269,7 @@
         <child>
           <object class="GtkButton" id="add_button">
             <property name="label" translatable="yes">Add</property>
-            <property name="visible">True</property>
+            <property name="visible" bind-source="headerbar" bind-property="show-close-button" bind-flags="default|sync-create|invert-boolean"/>
             <property name="can_focus">True</property>
             <property name="sensitive">False</property>
             <property name="receives_default">True</property>
diff --git a/src/gcal-source-dialog.c b/src/gcal-source-dialog.c
index a993753e..6e017e11 100644
--- a/src/gcal-source-dialog.c
+++ b/src/gcal-source-dialog.c
@@ -714,10 +714,6 @@ gcal_source_dialog_set_mode (GcalSourceDialog    *dialog,
   edit_mode = (mode == GCAL_SOURCE_DIALOG_MODE_EDIT);
 
   gtk_header_bar_set_show_close_button (GTK_HEADER_BAR (priv->headerbar), edit_mode);
-  gtk_widget_set_visible (priv->cancel_button, !edit_mode);
-  gtk_widget_set_visible (priv->add_button, !edit_mode);
-  gtk_widget_set_visible (priv->edit_grid, edit_mode);
-  gtk_widget_set_visible (priv->notebook, !edit_mode);
   gtk_stack_set_visible_child_name (GTK_STACK (priv->stack), edit_mode ? "edit" : "create");
 
   if (!edit_mode)