抓取数据逻辑优化；减少次数，提高速率，节约流量

lib/db/getSiteData.js

@@ -7,7 +7,7 @@ import { getConfigMapFromConfigPage } from '@/lib/notion/getNotionConfig'
 import getPageProperties, {
   adjustPageProperties
 } from '@/lib/notion/getPageProperties'
-import { getPostBlocks, getSingleBlock } from '@/lib/notion/getPostBlocks'
+import { fetchInBatches, getPostBlocks } from '@/lib/notion/getPostBlocks'
 import { compressImage, mapImgUrl } from '@/lib/notion/mapImage'
 import { deepClone } from '@/lib/utils'
 import { idToUuid } from 'notion-utils'
@@ -371,13 +371,14 @@ const EmptyData = pageId => {
  * @returns {Promise<JSX.Element|null|*>}
  */
 async function getDataBaseInfoByNotionAPI({ pageId, from }) {
+  console.log('[Fetching Data]', pageId, from)
   const pageRecordMap = await getPostBlocks(pageId, from)
   if (!pageRecordMap) {
     console.error('can`t get Notion Data ; Which id is: ', pageId)
     return {}
   }
   pageId = idToUuid(pageId)
-  const block = pageRecordMap.block || {}
+  let block = pageRecordMap.block || {}
   const rawMetadata = block[pageId]?.value
   // Check Type Page-Database和Inline-Database
   if (
@@ -402,6 +403,7 @@ async function getDataBaseInfoByNotionAPI({ pageId, from }) {
     collectionView,
     viewIds
   )
+
   if (pageIds?.length === 0) {
     console.error(
       '获取到的文章列表为空，请检查notion模板',
@@ -415,29 +417,22 @@ async function getDataBaseInfoByNotionAPI({ pageId, from }) {
     // console.log('有效Page数量', pageIds?.length)
   }
 
-  // 获取每篇文章基础数据
+  // 抓取主数据库最多抓取1000个blocks，溢出的数block这里统一抓取一遍
+  const blockIdsNeedFetch = []
   for (let i = 0; i < pageIds.length; i++) {
     const id = pageIds[i]
     const value = block[id]?.value
     if (!value) {
-      // 如果找不到文章对应的block，说明发生了溢出，使用pageID再去请求
-      const pageBlock = await getSingleBlock(id, from)
-      if (pageBlock.block[id].value) {
-        const properties =
-          (await getPageProperties(
-            id,
-            pageBlock.block[id].value,
-            schema,
-            null,
-            getTagOptions(schema)
-          )) || null
-        if (properties) {
-          collectionData.push(properties)
-        }
-      }
-      continue
+      blockIdsNeedFetch.push(id)
     }
+  }
+  const fetchedBlocks = await fetchInBatches(blockIdsNeedFetch)
+  block = Object.assign({}, block, fetchedBlocks)
 
+  // 获取每篇文章基础数据
+  for (let i = 0; i < pageIds.length; i++) {
+    const id = pageIds[i]
+    const value = block[id]?.value || fetchedBlocks[id]?.value
     const properties =
       (await getPageProperties(
         id,
@@ -446,6 +441,7 @@ async function getDataBaseInfoByNotionAPI({ pageId, from }) {
         null,
         getTagOptions(schema)
       )) || null
+
     if (properties) {
       collectionData.push(properties)
     }