diff --git a/lib/db/getSiteData.js b/lib/db/getSiteData.js index 4d615824..cefc25f0 100755 --- a/lib/db/getSiteData.js +++ b/lib/db/getSiteData.js @@ -1,5 +1,4 @@ import BLOG from '@/blog.config' -import { getDataFromCache, setDataToCache } from '@/lib/cache/cache_manager' import { getAllCategories } from '@/lib/notion/getAllCategories' import getAllPageIds from '@/lib/notion/getAllPageIds' import { getAllTags } from '@/lib/notion/getAllTags' @@ -42,7 +41,7 @@ export async function getGlobalData({ const prefix = extractLangPrefix(siteId) // 第一个id站点默认语言 if (index === 0 || locale === prefix) { - data = await getNotionPageData({ + data = await getSiteDataByPageId({ pageId: id, from }) @@ -60,24 +59,241 @@ export async function getGlobalData({ * @param from 请求来源 * @returns {Promise} */ -export async function getNotionPageData({ pageId, from }) { - // 尝试从缓存获取 - const cacheKey = 'page_block_' + pageId - let data = await getDataFromCache(cacheKey) - if (data && data.pageIds?.length > 0) { - console.debug('[API<<--缓存]', `from:${from}`, `root-page-id:${pageId}`) - return data +export async function getSiteDataByPageId({ pageId, from }) { + // 获取NOTION原始数据,此接支持mem缓存。 + const pageRecordMap = await getPage(pageId, from) + // 将Notion数据按规则转成站点数据 + const data = await converNotionToSiteDate(pageId, from, pageRecordMap) + return data +} + +/** + * 获取公告 + */ +async function getNotice(post) { + if (!post) { + return null + } + + post.blockMap = await getPage(post.id, 'data-notice') + return post +} + +/** + * 空的默认数据 + * @param {*} pageId + * @returns + */ +const EmptyData = pageId => { + const empty = { + notice: null, + siteInfo: getSiteInfo({}), + allPages: [ + { + id: 1, + title: `无法获取Notion数据,请检查Notion_ID: \n 当前 ${pageId}`, + summary: + '访问文档获取帮助→ https://tangly1024.com/article/vercel-deploy-notion-next', + status: 'Published', + type: 'Post', + slug: '13a171332816461db29d50e9f575b00d', + publishDay: '2024-11-13', + pageCoverThumbnail: BLOG.HOME_BANNER_IMAGE, + date: { + start_date: '2023-04-24', + lastEditedDay: '2023-04-24', + tagItems: [] + } + } + ], + allNavPages: [], + collection: [], + collectionQuery: {}, + collectionId: null, + collectionView: {}, + viewIds: [], + block: {}, + schema: {}, + tagOptions: [], + categoryOptions: [], + rawMetadata: {}, + customNav: [], + customMenu: [], + postCount: 1, + pageIds: [], + latestPosts: [] + } + return empty +} + +/** + * 将Notion数据转站点数据 + * 这里统一对数据格式化 + * @returns {Promise} + */ +async function converNotionToSiteDate(pageId, from, pageRecordMap) { + if (!pageRecordMap) { + console.error('can`t get Notion Data ; Which id is: ', pageId) + return {} + } + pageId = idToUuid(pageId) + let block = pageRecordMap.block || {} + const rawMetadata = block[pageId]?.value + // Check Type Page-Database和Inline-Database + if ( + rawMetadata?.type !== 'collection_view_page' && + rawMetadata?.type !== 'collection_view' + ) { + console.error(`pageId "${pageId}" is not a database`) + return EmptyData(pageId) + } + const collection = Object.values(pageRecordMap.collection)[0]?.value || {} + const collectionId = rawMetadata?.collection_id + const collectionQuery = pageRecordMap.collection_query + const collectionView = pageRecordMap.collection_view + const schema = collection?.schema + + const viewIds = rawMetadata?.view_ids + const collectionData = [] + + const pageIds = getAllPageIds( + collectionQuery, + collectionId, + collectionView, + viewIds + ) + + if (pageIds?.length === 0) { + console.error( + '获取到的文章列表为空,请检查notion模板', + collectionQuery, + collection, + collectionView, + viewIds, + pageRecordMap + ) } else { - // 从接口读取 - data = await getDataBaseInfoByNotionAPI({ pageId, from }) - // 存入缓存 - if (data) { - await setDataToCache(cacheKey, data) + // console.log('有效Page数量', pageIds?.length) + } + + // 抓取主数据库最多抓取1000个blocks,溢出的数block这里统一抓取一遍 + const blockIdsNeedFetch = [] + for (let i = 0; i < pageIds.length; i++) { + const id = pageIds[i] + const value = block[id]?.value + if (!value) { + blockIdsNeedFetch.push(id) + } + } + const fetchedBlocks = await fetchInBatches(blockIdsNeedFetch) + block = Object.assign({}, block, fetchedBlocks) + + // 获取每篇文章基础数据 + for (let i = 0; i < pageIds.length; i++) { + const id = pageIds[i] + const value = block[id]?.value || fetchedBlocks[id]?.value + const properties = + (await getPageProperties( + id, + value, + schema, + null, + getTagOptions(schema) + )) || null + + if (properties) { + collectionData.push(properties) } } - // 返回给前端的数据做处理 - return data + // 站点配置优先读取配置表格,否则读取blog.config.js 文件 + const NOTION_CONFIG = (await getConfigMapFromConfigPage(collectionData)) || {} + + // 处理每一条数据的字段 + collectionData.forEach(function (element) { + adjustPageProperties(element, NOTION_CONFIG) + }) + + // 站点基础信息 + const siteInfo = getSiteInfo({ collection, block, pageId }) + + // 文章计数 + let postCount = 0 + + // 查找所有的Post和Page + const allPages = collectionData.filter(post => { + if (post?.type === 'Post' && post.status === 'Published') { + postCount++ + } + return ( + post && + post?.slug && + // !post?.slug?.startsWith('http') && + (post?.status === 'Invisible' || post?.status === 'Published') + ) + }) + + // Sort by date + if (siteConfig('POSTS_SORT_BY', '', NOTION_CONFIG) === 'date') { + allPages.sort((a, b) => { + return b?.publishDate - a?.publishDate + }) + } + + const notice = await getNotice( + collectionData.filter(post => { + return ( + post && + post?.type && + post?.type === 'Notice' && + post.status === 'Published' + ) + })?.[0] + ) + // 所有分类 + const categoryOptions = getAllCategories({ + allPages, + categoryOptions: getCategoryOptions(schema) + }) + // 所有标签 + const tagOptions = getAllTags({ + allPages, + tagOptions: getTagOptions(schema), + NOTION_CONFIG + }) + // 旧的菜单 + const customNav = getCustomNav({ + allPages: collectionData.filter( + post => post?.type === 'Page' && post.status === 'Published' + ) + }) + // 新的菜单 + const customMenu = await getCustomMenu({ collectionData, NOTION_CONFIG }) + const latestPosts = getLatestPosts({ allPages, from, latestPostCount: 6 }) + const allNavPages = getNavPages({ allPages }) + + return { + NOTION_CONFIG, + notice, + siteInfo, + allPages, + allNavPages, + collection, + collectionQuery, + collectionId, + collectionView, + viewIds, + block, + schema, + tagOptions, + categoryOptions, + rawMetadata, + customNav, + customMenu, + postCount, + pageIds, + latestPosts + } } /** @@ -404,228 +620,3 @@ export function getNavPages({ allPages }) { ext: item.ext || {} })) } - -/** - * 获取公告 - */ -async function getNotice(post) { - if (!post) { - return null - } - - post.blockMap = await getPage(post.id, 'data-notice') - return post -} - -// 没有数据时返回 -const EmptyData = pageId => { - const empty = { - notice: null, - siteInfo: getSiteInfo({}), - allPages: [ - { - id: 1, - title: `无法获取Notion数据,请检查Notion_ID: \n 当前 ${pageId}`, - summary: - '访问文档获取帮助→ https://tangly1024.com/article/vercel-deploy-notion-next', - status: 'Published', - type: 'Post', - slug: '13a171332816461db29d50e9f575b00d', - pageCoverThumbnail: BLOG.HOME_BANNER_IMAGE, - date: { - start_date: '2023-04-24', - lastEditedDay: '2023-04-24', - tagItems: [] - } - } - ], - allNavPages: [], - collection: [], - collectionQuery: {}, - collectionId: null, - collectionView: {}, - viewIds: [], - block: {}, - schema: {}, - tagOptions: [], - categoryOptions: [], - rawMetadata: {}, - customNav: [], - customMenu: [], - postCount: 1, - pageIds: [], - latestPosts: [] - } - return empty -} - -/** - * 调用NotionAPI获取Page数据 - * @returns {Promise} - */ -async function getDataBaseInfoByNotionAPI({ pageId, from }) { - console.log('[Fetching Data]', pageId, from) - const pageRecordMap = await getPage(pageId, from) - if (!pageRecordMap) { - console.error('can`t get Notion Data ; Which id is: ', pageId) - return {} - } - pageId = idToUuid(pageId) - let block = pageRecordMap.block || {} - const rawMetadata = block[pageId]?.value - // Check Type Page-Database和Inline-Database - if ( - rawMetadata?.type !== 'collection_view_page' && - rawMetadata?.type !== 'collection_view' - ) { - console.error(`pageId "${pageId}" is not a database`) - return EmptyData(pageId) - } - const collection = Object.values(pageRecordMap.collection)[0]?.value || {} - const collectionId = rawMetadata?.collection_id - const collectionQuery = pageRecordMap.collection_query - const collectionView = pageRecordMap.collection_view - const schema = collection?.schema - - const viewIds = rawMetadata?.view_ids - const collectionData = [] - - const pageIds = getAllPageIds( - collectionQuery, - collectionId, - collectionView, - viewIds - ) - - if (pageIds?.length === 0) { - console.error( - '获取到的文章列表为空,请检查notion模板', - collectionQuery, - collection, - collectionView, - viewIds, - pageRecordMap - ) - } else { - // console.log('有效Page数量', pageIds?.length) - } - - // 抓取主数据库最多抓取1000个blocks,溢出的数block这里统一抓取一遍 - const blockIdsNeedFetch = [] - for (let i = 0; i < pageIds.length; i++) { - const id = pageIds[i] - const value = block[id]?.value - if (!value) { - blockIdsNeedFetch.push(id) - } - } - const fetchedBlocks = await fetchInBatches(blockIdsNeedFetch) - block = Object.assign({}, block, fetchedBlocks) - - // 获取每篇文章基础数据 - for (let i = 0; i < pageIds.length; i++) { - const id = pageIds[i] - const value = block[id]?.value || fetchedBlocks[id]?.value - const properties = - (await getPageProperties( - id, - value, - schema, - null, - getTagOptions(schema) - )) || null - - if (properties) { - collectionData.push(properties) - } - } - - // 站点配置优先读取配置表格,否则读取blog.config.js 文件 - const NOTION_CONFIG = (await getConfigMapFromConfigPage(collectionData)) || {} - - // 处理每一条数据的字段 - collectionData.forEach(function (element) { - adjustPageProperties(element, NOTION_CONFIG) - }) - - // 站点基础信息 - const siteInfo = getSiteInfo({ collection, block, pageId }) - - // 文章计数 - let postCount = 0 - - // 查找所有的Post和Page - const allPages = collectionData.filter(post => { - if (post?.type === 'Post' && post.status === 'Published') { - postCount++ - } - return ( - post && - post?.slug && - // !post?.slug?.startsWith('http') && - (post?.status === 'Invisible' || post?.status === 'Published') - ) - }) - - // Sort by date - if (siteConfig('POSTS_SORT_BY', '', NOTION_CONFIG) === 'date') { - allPages.sort((a, b) => { - return b?.publishDate - a?.publishDate - }) - } - - const notice = await getNotice( - collectionData.filter(post => { - return ( - post && - post?.type && - post?.type === 'Notice' && - post.status === 'Published' - ) - })?.[0] - ) - // 所有分类 - const categoryOptions = getAllCategories({ - allPages, - categoryOptions: getCategoryOptions(schema) - }) - // 所有标签 - const tagOptions = getAllTags({ - allPages, - tagOptions: getTagOptions(schema), - NOTION_CONFIG - }) - // 旧的菜单 - const customNav = getCustomNav({ - allPages: collectionData.filter( - post => post?.type === 'Page' && post.status === 'Published' - ) - }) - // 新的菜单 - const customMenu = await getCustomMenu({ collectionData, NOTION_CONFIG }) - const latestPosts = getLatestPosts({ allPages, from, latestPostCount: 6 }) - const allNavPages = getNavPages({ allPages }) - - return { - NOTION_CONFIG, - notice, - siteInfo, - allPages, - allNavPages, - collection, - collectionQuery, - collectionId, - collectionView, - viewIds, - block, - schema, - tagOptions, - categoryOptions, - rawMetadata, - customNav, - customMenu, - postCount, - pageIds, - latestPosts - } -} diff --git a/lib/notion/getNotionConfig.js b/lib/notion/getNotionConfig.js index c0ef7c29..ae987369 100644 --- a/lib/notion/getNotionConfig.js +++ b/lib/notion/getNotionConfig.js @@ -157,7 +157,8 @@ export async function getConfigMapFromConfigPage(allPages) { // 只导入生效的配置 if (config.enable) { // console.log('[Notion配置]', config.key, config.value) - notionConfig[config.key] = config.value + notionConfig[config.key] = config.value || '' + // 配置不能是undefined,至少是空字符串 } } } diff --git a/lib/notion/getPageProperties.js b/lib/notion/getPageProperties.js index d62568cd..27d14047 100644 --- a/lib/notion/getPageProperties.js +++ b/lib/notion/getPageProperties.js @@ -180,9 +180,7 @@ export function adjustPageProperties(properties, NOTION_CONFIG) { // 1.按照用户配置的URL_PREFIX 转换一下slug // 2.为文章添加一个href字段,存储最终调整的路径 if (properties.type === 'Post') { - if (siteConfig('POST_URL_PREFIX', '', NOTION_CONFIG)) { - properties.slug = generateCustomizeSlug(properties, NOTION_CONFIG) - } + properties.slug = generateCustomizeSlug(properties, NOTION_CONFIG) properties.href = properties.slug ?? properties.id } else if (properties.type === 'Page') { properties.href = properties.slug ?? properties.id @@ -245,11 +243,9 @@ function generateCustomizeSlug(postProperties, NOTION_CONFIG) { return postProperties.slug } let fullPrefix = '' - const allSlugPatterns = siteConfig( - 'POST_URL_PREFIX', - '', - NOTION_CONFIG - ).split('/') + const allSlugPatterns = + NOTION_CONFIG?.POST_URL_PREFIX || + siteConfig('POST_URL_PREFIX', '', NOTION_CONFIG).split('/') const POST_URL_PREFIX_MAPPING_CATEGORY = siteConfig( 'POST_URL_PREFIX_MAPPING_CATEGORY', diff --git a/lib/notion/getPostBlocks.js b/lib/notion/getPostBlocks.js index 2859345f..1f8733f5 100644 --- a/lib/notion/getPostBlocks.js +++ b/lib/notion/getPostBlocks.js @@ -10,19 +10,20 @@ import { deepClone, delay } from '../utils' * @param {*} slice * @returns */ -export async function getPage(id, from, slice) { - const cacheKey = 'page_block_' + id +export async function getPage(id, from = null, slice) { + const cacheKey = `page_block_${id}` let pageBlock = await getDataFromCache(cacheKey) if (pageBlock) { - // console.log('[API<<--缓存]', `from:${from}`, cacheKey) - return filterPostBlocks(id, pageBlock, slice) + // console.debug('[API<<--缓存]', `from:${from}`, cacheKey) + return convertNotionBlocksToPost(id, pageBlock, slice) } + // 抓取最新数据 pageBlock = await getPageWithRetry(id, from) if (pageBlock) { await setDataToCache(cacheKey, pageBlock) - return filterPostBlocks(id, pageBlock, slice) + return convertNotionBlocksToPost(id, pageBlock, slice) } return pageBlock } @@ -69,7 +70,7 @@ export async function getPageWithRetry(id, from, retryAttempts = 3) { } /** - * 获取到的页面BLOCK特殊处理 + * Notion页面BLOCK格式化处理 * 1.删除冗余字段 * 2.比如文件、视频、音频、url格式化 * 3.代码块等元素兼容 @@ -78,72 +79,72 @@ export async function getPageWithRetry(id, from, retryAttempts = 3) { * @param {*} slice 截取数量 * @returns */ -function filterPostBlocks(id, blockMap, slice) { +function convertNotionBlocksToPost(id, blockMap, slice) { const clonePageBlock = deepClone(blockMap) let count = 0 const blocksToProcess = Object.keys(clonePageBlock?.block || {}) // 循环遍历文档的每个block for (let i = 0; i < blocksToProcess.length; i++) { - const blockId = blocksToProcess[i] - const b = clonePageBlock?.block[blockId] - - if (slice && slice > 0 && count > slice) { - delete clonePageBlock?.block[blockId] - continue - } - - // 当BlockId等于PageId时移除 - if (b?.value?.id === id) { - // 此block含有敏感信息 - delete b?.value?.properties - continue - } - - count++ - - if (b?.value?.type === 'sync_block' && b?.value?.children) { - const childBlocks = b.value.children - // 移除同步块 - delete clonePageBlock.block[blockId] - // 用子块替代同步块 - childBlocks.forEach((childBlock, index) => { - const newBlockId = `${blockId}_child_${index}` - clonePageBlock.block[newBlockId] = childBlock - blocksToProcess.splice(i + index + 1, 0, newBlockId) - }) - // 重新处理新加入的子块 - i-- - continue - } - - // 处理 c++、c#、汇编等语言名字映射 - if (b?.value?.type === 'code') { - if (b?.value?.properties?.language?.[0][0] === 'C++') { - b.value.properties.language[0][0] = 'cpp' + const blockId = blocksToProcess[i] + const b = clonePageBlock?.block[blockId] + + if (slice && slice > 0 && count > slice) { + delete clonePageBlock?.block[blockId] + continue } - if (b?.value?.properties?.language?.[0][0] === 'C#') { - b.value.properties.language[0][0] = 'csharp' + + // 当BlockId等于PageId时移除 + if (b?.value?.id === id) { + // 此block含有敏感信息 + delete b?.value?.properties + continue } - if (b?.value?.properties?.language?.[0][0] === 'Assembly') { - b.value.properties.language[0][0] = 'asm6502' + + count++ + + if (b?.value?.type === 'sync_block' && b?.value?.children) { + const childBlocks = b.value.children + // 移除同步块 + delete clonePageBlock.block[blockId] + // 用子块替代同步块 + childBlocks.forEach((childBlock, index) => { + const newBlockId = `${blockId}_child_${index}` + clonePageBlock.block[newBlockId] = childBlock + blocksToProcess.splice(i + index + 1, 0, newBlockId) + }) + // 重新处理新加入的子块 + i-- + continue + } + + // 处理 c++、c#、汇编等语言名字映射 + if (b?.value?.type === 'code') { + if (b?.value?.properties?.language?.[0][0] === 'C++') { + b.value.properties.language[0][0] = 'cpp' + } + if (b?.value?.properties?.language?.[0][0] === 'C#') { + b.value.properties.language[0][0] = 'csharp' + } + if (b?.value?.properties?.language?.[0][0] === 'Assembly') { + b.value.properties.language[0][0] = 'asm6502' + } + } + + // 如果是文件,或嵌入式PDF,需要重新加密签名 + if ( + (b?.value?.type === 'file' || + b?.value?.type === 'pdf' || + b?.value?.type === 'video' || + b?.value?.type === 'audio') && + b?.value?.properties?.source?.[0][0] && + b?.value?.properties?.source?.[0][0].indexOf('amazonaws.com') > 0 + ) { + const oldUrl = b?.value?.properties?.source?.[0][0] + const newUrl = `https://notion.so/signed/${encodeURIComponent(oldUrl)}?table=block&id=${b?.value?.id}` + b.value.properties.source[0][0] = newUrl } } - - // 如果是文件,或嵌入式PDF,需要重新加密签名 - if ( - (b?.value?.type === 'file' || - b?.value?.type === 'pdf' || - b?.value?.type === 'video' || - b?.value?.type === 'audio') && - b?.value?.properties?.source?.[0][0] && - b?.value?.properties?.source?.[0][0].indexOf('amazonaws.com') > 0 - ) { - const oldUrl = b?.value?.properties?.source?.[0][0] - const newUrl = `https://notion.so/signed/${encodeURIComponent(oldUrl)}?table=block&id=${b?.value?.id}` - b.value.properties.source[0][0] = newUrl - } -} // 去掉不用的字段 if (id === BLOG.NOTION_PAGE_ID) { diff --git a/lib/sitemap.xml.js b/lib/sitemap.xml.js index c520c0ff..55b85b9b 100644 --- a/lib/sitemap.xml.js +++ b/lib/sitemap.xml.js @@ -1,29 +1,34 @@ - -import fs from 'fs' import BLOG from '@/blog.config' +import fs from 'fs' export async function generateSitemapXml({ allPages }) { - const urls = [{ - loc: `${BLOG.LINK}`, - lastmod: new Date().toISOString().split('T')[0], - changefreq: 'daily' - }, { - loc: `${BLOG.LINK}/archive`, - lastmod: new Date().toISOString().split('T')[0], - changefreq: 'daily' - }, { - loc: `${BLOG.LINK}/category`, - lastmod: new Date().toISOString().split('T')[0], - changefreq: 'daily' - }, { - loc: `${BLOG.LINK}/tag`, - lastmod: new Date().toISOString().split('T')[0], - changefreq: 'daily' - }] - + const urls = [ + { + loc: `${BLOG.LINK}`, + lastmod: new Date().toISOString().split('T')[0], + changefreq: 'daily' + }, + { + loc: `${BLOG.LINK}/archive`, + lastmod: new Date().toISOString().split('T')[0], + changefreq: 'daily' + }, + { + loc: `${BLOG.LINK}/category`, + lastmod: new Date().toISOString().split('T')[0], + changefreq: 'daily' + }, + { + loc: `${BLOG.LINK}/tag`, + lastmod: new Date().toISOString().split('T')[0], + changefreq: 'daily' + } + ] // 循环页面生成 allPages?.forEach(post => { - const slugWithoutLeadingSlash = post?.slug?.startsWith('/') ? post?.slug?.slice(1) : post.slug + const slugWithoutLeadingSlash = post?.slug?.startsWith('/') + ? post?.slug?.slice(1) + : post.slug urls.push({ loc: `${BLOG.LINK}/${slugWithoutLeadingSlash}`, lastmod: new Date(post?.publishDay).toISOString().split('T')[0],