diff --git a/src/com/android/dialer/dialpad/DialpadFragment.java b/src/com/android/dialer/dialpad/DialpadFragment.java
index 123d700..3693e6a 100644
--- a/src/com/android/dialer/dialpad/DialpadFragment.java
+++ b/src/com/android/dialer/dialpad/DialpadFragment.java
@@ -79,7 +79,6 @@
 import com.android.dialer.DialtactsActivity;
 import com.android.dialer.R;
 import com.android.dialer.SpecialCharSequenceMgr;
-import com.android.dialer.dialpad.SmartDialCache.SmartDialContentObserver;
 import com.android.dialer.interactions.PhoneNumberInteraction;
 import com.android.dialer.util.OrientationUtil;
 import com.android.internal.telephony.ITelephony;
@@ -154,7 +153,6 @@
      * Will be set only if the view has the smart dialing section.
      */
     private SmartDialAdapter mSmartDialAdapter;
-    private SmartDialContentObserver mSmartDialObserver;
 
     /**
      * Regular expression prohibiting manual phone call. Can be empty, which means "no rule".
@@ -226,6 +224,7 @@
     private boolean mDigitsFilledByIntent;
 
     private boolean mStartedFromNewIntent = false;
+    private boolean mFirstLaunch = false;
 
     private static final String PREF_DIGITS_FILLED_BY_INTENT = "pref_digits_filled_by_intent";
 
@@ -280,7 +279,7 @@
     @Override
     public void onCreate(Bundle state) {
         super.onCreate(state);
-
+        mFirstLaunch = true;
         mContactsPrefs = new ContactsPreferences(getActivity());
         mCurrentCountryIso = GeoUtil.getCurrentCountryIso(getActivity());
         mSmartDialCache = SmartDialCache.getInstance(getActivity(),
@@ -300,10 +299,6 @@
         if (state != null) {
             mDigitsFilledByIntent = state.getBoolean(PREF_DIGITS_FILLED_BY_INTENT);
         }
-
-        mSmartDialObserver = new SmartDialContentObserver(new Handler(), mSmartDialCache);
-        this.getActivity().getContentResolver().registerContentObserver(
-                SmartDialCache.PhoneQuery.URI, true, mSmartDialObserver);
     }
 
     @Override
@@ -614,6 +609,19 @@
             showDialpadChooser(false);
         }
 
+        // Don't force recache if this is the first time onResume is being called, since caching
+        // should already happen in setUserVisibleHint.
+        if (!mFirstLaunch) {
+            // This forced recache covers the case where the dialer was previously running, and
+            // was brought back into the foreground. If the dialpad fragment hasn't actually
+            // become visible throughout the entire activity's lifecycle, it is possible that
+            // caching hasn't happened yet. In this case, we can force a recache anyway, since we
+            // are not worried about startup performance anymore.
+            mSmartDialCache.cacheIfNeeded(true);
+        }
+
+        mFirstLaunch = false;
+
         stopWatch.lap("hnt");
 
         updateDialAndDeleteButtonEnabledState();
@@ -621,9 +629,6 @@
         stopWatch.lap("bes");
 
         stopWatch.stopAndLog(TAG, 50);
-
-        this.getActivity().getContentResolver().registerContentObserver(
-                SmartDialCache.PhoneQuery.URI, true, mSmartDialObserver);
     }
 
     @Override
@@ -651,8 +656,6 @@
         mLastNumberDialed = EMPTY_NUMBER;  // Since we are going to query again, free stale number.
 
         SpecialCharSequenceMgr.cleanup();
-
-        getActivity().getContentResolver().unregisterContentObserver(mSmartDialObserver);
     }
 
     @Override
@@ -1661,6 +1664,10 @@
     public void setUserVisibleHint(boolean isVisibleToUser) {
         super.setUserVisibleHint(isVisibleToUser);
         if (isVisibleToUser) {
+            // This is called if the dialpad fragment is swiped into to view for the very first
+            // time in the activity's lifecycle, or the user starts the dialer for the first time
+            // and the dialpad fragment is displayed immediately, and is what causes the initial
+            // caching process to happen.
             mSmartDialCache.cacheIfNeeded(false);
         }
     }
diff --git a/src/com/android/dialer/dialpad/SmartDialCache.java b/src/com/android/dialer/dialpad/SmartDialCache.java
index dbec5a6..4f4ce14 100644
--- a/src/com/android/dialer/dialpad/SmartDialCache.java
+++ b/src/com/android/dialer/dialpad/SmartDialCache.java
@@ -243,8 +243,8 @@
 
     /**
      * Cache contacts only if there is a need to (forced cache refresh or no attempt to cache yet).
-     * This method is called in 2 places: whenever the DialpadFragment comes into view, and when the
-     * ContentObserver observes a change in contacts.
+     * This method is called in 2 places: whenever the DialpadFragment comes into view, and in
+     * onResume.
      *
      * @param forceRecache If true, force a cache refresh.
      */
@@ -284,32 +284,4 @@
         }
 
     }
-
-    public static class SmartDialContentObserver extends ContentObserver {
-        private final SmartDialCache mCache;
-        // throttle updates in case onChange is called too often due to syncing, etc.
-        private final long mThresholdBetweenUpdates = 5000;
-        private long mLastCalled = 0;
-        private long mLastUpdated = 0;
-        public SmartDialContentObserver(Handler handler, SmartDialCache cache) {
-            super(handler);
-            mCache = cache;
-        }
-
-        @Override
-        public void onChange(boolean selfChange) {
-            mLastCalled = System.currentTimeMillis();
-            if (DEBUG) {
-                Log.d(LOG_TAG, "Contacts change observed");
-            }
-            if (mLastCalled - mLastUpdated > mThresholdBetweenUpdates) {
-                mLastUpdated = mLastCalled;
-                if (DEBUG) {
-                    Log.d(LOG_TAG, "More than 5 seconds since last cache, forcing recache");
-                }
-                mCache.cacheIfNeeded(true);
-            }
-            super.onChange(selfChange);
-        }
-    }
 }