diff --git a/plugins/smartcard/gsd-smartcard-manager.c b/plugins/smartcard/gsd-smartcard-manager.c
index 94ffdd9..09ccecc 100644
--- a/plugins/smartcard/gsd-smartcard-manager.c
+++ b/plugins/smartcard/gsd-smartcard-manager.c
@@ -184,7 +184,7 @@ watch_one_event_from_driver (GsdSmartcardManager       *self,
                              GError                   **error)
 {
         GsdSmartcardManagerPrivate *priv = self->priv;
-        PK11SlotInfo *card, *old_card;
+        PK11SlotInfo *card = NULL, *old_card;
         CK_SLOT_ID slot_id;
         gulong handler_id;
         int old_slot_series = -1, slot_series;
@@ -194,7 +194,8 @@ watch_one_event_from_driver (GsdSmartcardManager       *self,
                                             operation,
                                             NULL);
 
-        card = SECMOD_WaitForAnyTokenEvent (operation->driver, 0, PR_SecondsToInterval (1));
+        if (handler_id != 0)
+                card = SECMOD_WaitForAnyTokenEvent (operation->driver, 0, PR_SecondsToInterval (1));
 
         g_cancellable_disconnect (cancellable, handler_id);
 
@@ -791,6 +792,8 @@ gsd_smartcard_manager_stop (GsdSmartcardManager *self)
 
         g_debug ("Stopping smartcard manager");
 
+        g_cancellable_cancel (priv->cancellable);
+
         unload_nss (self);
 
         g_clear_object (&priv->settings);