Avoid searching for the target catcache in CatalogCacheIdInvalidate.

A test case provided by Mathieu Fenniak shows that the initial search for
the target catcache in CatalogCacheIdInvalidate consumes a very significant
amount of overhead in cases where cache invalidation is triggered but has
little useful work to do.  There is no good reason for that search to exist
at all, as the index array maintained by syscache.c allows direct lookup of
the catcache from its ID.  We just need a frontend function in syscache.c,
matching the division of labor for most other cache-accessing operations.

While there's more that can be done in this area, this patch alone reduces
the runtime of Mathieu's example by 2X.  We can hope that it offers some
useful benefit in other cases too, although usually cache invalidation
overhead is not such a striking fraction of the total runtime.

Back-patch to 9.4 where logical decoding was introduced.  It might be
worth going further back, but presently the only case we know of where
cache invalidation is really a significant burden is in logical decoding.
Also, older branches have fewer catcaches, reducing the possible benefit.

(Note: although this nominally changes catcache's API, we have always
documented CatalogCacheIdInvalidate as a private function, so I would
have little sympathy for an external module calling it directly.  So
backpatching should be fine.)

Discussion: https://postgr.es/m/CAHoiPjzea6N0zuCi=+f9v_j94nfsy6y8SU7-=bp4=7qw6_i=Rg@mail.gmail.com

src/backend/utils/cache/catcache.c

@@ -427,7 +427,7 @@ CatCacheRemoveCList(CatCache *cache, CatCList *cl)
 
 
 /*
- *	CatalogCacheIdInvalidate
+ *	CatCacheInvalidate
  *
  *	Invalidate entries in the specified cache, given a hash value.
  *
@@ -445,71 +445,57 @@ CatCacheRemoveCList(CatCache *cache, CatCList *cl)
  *	This routine is only quasi-public: it should only be used by inval.c.
  */
 void
-CatalogCacheIdInvalidate(int cacheId, uint32 hashValue)
+CatCacheInvalidate(CatCache *cache, uint32 hashValue)
 {
-	slist_iter	cache_iter;
+	Index		hashIndex;
+	dlist_mutable_iter iter;
 
-	CACHE1_elog(DEBUG2, "CatalogCacheIdInvalidate: called");
+	CACHE1_elog(DEBUG2, "CatCacheInvalidate: called");
 
 	/*
-	 * inspect caches to find the proper cache
+	 * We don't bother to check whether the cache has finished initialization
+	 * yet; if not, there will be no entries in it so no problem.
 	 */
-	slist_foreach(cache_iter, &CacheHdr->ch_caches)
+
+	/*
+	 * Invalidate *all* CatCLists in this cache; it's too hard to tell which
+	 * searches might still be correct, so just zap 'em all.
+	 */
+	dlist_foreach_modify(iter, &cache->cc_lists)
 	{
-		CatCache   *ccp = slist_container(CatCache, cc_next, cache_iter.cur);
-		Index		hashIndex;
-		dlist_mutable_iter iter;
+		CatCList   *cl = dlist_container(CatCList, cache_elem, iter.cur);
 
-		if (cacheId != ccp->id)
-			continue;
+		if (cl->refcount > 0)
+			cl->dead = true;
+		else
+			CatCacheRemoveCList(cache, cl);
+	}
 
-		/*
-		 * We don't bother to check whether the cache has finished
-		 * initialization yet; if not, there will be no entries in it so no
-		 * problem.
-		 */
+	/*
+	 * inspect the proper hash bucket for tuple matches
+	 */
+	hashIndex = HASH_INDEX(hashValue, cache->cc_nbuckets);
+	dlist_foreach_modify(iter, &cache->cc_bucket[hashIndex])
+	{
+		CatCTup    *ct = dlist_container(CatCTup, cache_elem, iter.cur);
 
-		/*
-		 * Invalidate *all* CatCLists in this cache; it's too hard to tell
-		 * which searches might still be correct, so just zap 'em all.
-		 */
-		dlist_foreach_modify(iter, &ccp->cc_lists)
+		if (hashValue == ct->hash_value)
 		{
-			CatCList   *cl = dlist_container(CatCList, cache_elem, iter.cur);
-
-			if (cl->refcount > 0)
-				cl->dead = true;
-			else
-				CatCacheRemoveCList(ccp, cl);
-		}
-
-		/*
-		 * inspect the proper hash bucket for tuple matches
-		 */
-		hashIndex = HASH_INDEX(hashValue, ccp->cc_nbuckets);
-		dlist_foreach_modify(iter, &ccp->cc_bucket[hashIndex])
-		{
-			CatCTup    *ct = dlist_container(CatCTup, cache_elem, iter.cur);
-
-			if (hashValue == ct->hash_value)
+			if (ct->refcount > 0 ||
+				(ct->c_list && ct->c_list->refcount > 0))
 			{
-				if (ct->refcount > 0 ||
-					(ct->c_list && ct->c_list->refcount > 0))
-				{
-					ct->dead = true;
-					/* list, if any, was marked dead above */
-					Assert(ct->c_list == NULL || ct->c_list->dead);
-				}
-				else
-					CatCacheRemoveCTup(ccp, ct);
-				CACHE1_elog(DEBUG2, "CatalogCacheIdInvalidate: invalidated");
-#ifdef CATCACHE_STATS
-				ccp->cc_invals++;
-#endif
-				/* could be multiple matches, so keep looking! */
+				ct->dead = true;
+				/* list, if any, was marked dead above */
+				Assert(ct->c_list == NULL || ct->c_list->dead);
 			}
+			else
+				CatCacheRemoveCTup(cache, ct);
+			CACHE1_elog(DEBUG2, "CatCacheInvalidate: invalidated");
+#ifdef CATCACHE_STATS
+			cache->cc_invals++;
+#endif
+			/* could be multiple matches, so keep looking! */
 		}
-		break;					/* need only search this one cache */
 	}
 }
 
@@ -1830,7 +1816,7 @@ build_dummy_tuple(CatCache *cache, int nkeys, ScanKey skeys)
  *	the specified relation, find all catcaches it could be in, compute the
  *	correct hash value for each such catcache, and call the specified
  *	function to record the cache id and hash value in inval.c's lists.
- *	CatalogCacheIdInvalidate will be called later, if appropriate,
+ *	SysCacheInvalidate will be called later, if appropriate,
  *	using the recorded information.
  *
  *	For an insert or delete, tuple is the target tuple and newtuple is NULL.

src/backend/utils/cache/inval.c

@@ -543,7 +543,7 @@ LocalExecuteInvalidationMessage(SharedInvalidationMessage *msg)
 		{
 			InvalidateCatalogSnapshot();
 
-			CatalogCacheIdInvalidate(msg->cc.id, msg->cc.hashValue);
+			SysCacheInvalidate(msg->cc.id, msg->cc.hashValue);
 
 			CallSyscacheCallbacks(msg->cc.id, msg->cc.hashValue);
 		}

src/backend/utils/cache/syscache.c

@@ -1211,6 +1211,27 @@ SearchSysCacheList(int cacheId, int nkeys,
 							  key1, key2, key3, key4);
 }
 
+/*
+ * SysCacheInvalidate
+ *
+ *	Invalidate entries in the specified cache, given a hash value.
+ *	See CatCacheInvalidate() for more info.
+ *
+ *	This routine is only quasi-public: it should only be used by inval.c.
+ */
+void
+SysCacheInvalidate(int cacheId, uint32 hashValue)
+{
+	if (cacheId < 0 || cacheId >= SysCacheSize)
+		elog(ERROR, "invalid cache ID: %d", cacheId);
+
+	/* if this cache isn't initialized yet, no need to do anything */
+	if (!PointerIsValid(SysCache[cacheId]))
+		return;
+
+	CatCacheInvalidate(SysCache[cacheId], hashValue);
+}
+
 /*
  * Certain relations that do not have system caches send snapshot invalidation
  * messages in lieu of catcache messages.  This is for the benefit of

src/include/utils/catcache.h

@@ -185,7 +185,7 @@ extern void ReleaseCatCacheList(CatCList *list);
 
 extern void ResetCatalogCaches(void);
 extern void CatalogCacheFlushCatalog(Oid catId);
-extern void CatalogCacheIdInvalidate(int cacheId, uint32 hashValue);
+extern void CatCacheInvalidate(CatCache *cache, uint32 hashValue);
 extern void PrepareToInvalidateCacheTuple(Relation relation,
 							  HeapTuple tuple,
 							  HeapTuple newtuple,

src/include/utils/syscache.h

@@ -129,6 +129,8 @@ struct catclist;
 extern struct catclist *SearchSysCacheList(int cacheId, int nkeys,
 				   Datum key1, Datum key2, Datum key3, Datum key4);
 
+extern void SysCacheInvalidate(int cacheId, uint32 hashValue);
+
 extern bool RelationInvalidatesSnapshotsOnly(Oid relid);
 extern bool RelationHasSysCache(Oid relid);
 extern bool RelationSupportsSysCache(Oid relid);