代码缓存逻辑调整

This commit is contained in:
tangly1024.com
2024-11-13 10:50:17 +08:00
parent 4798b5b422
commit eed08777d1
5 changed files with 327 additions and 333 deletions

View File

@@ -1,5 +1,4 @@
import BLOG from '@/blog.config'
import { getDataFromCache, setDataToCache } from '@/lib/cache/cache_manager'
import { getAllCategories } from '@/lib/notion/getAllCategories'
import getAllPageIds from '@/lib/notion/getAllPageIds'
import { getAllTags } from '@/lib/notion/getAllTags'
@@ -42,7 +41,7 @@ export async function getGlobalData({
const prefix = extractLangPrefix(siteId)
// 第一个id站点默认语言
if (index === 0 || locale === prefix) {
data = await getNotionPageData({
data = await getSiteDataByPageId({
pageId: id,
from
})
@@ -60,24 +59,241 @@ export async function getGlobalData({
* @param from 请求来源
* @returns {Promise<JSX.Element|*|*[]>}
*/
export async function getNotionPageData({ pageId, from }) {
// 尝试从缓存获取
const cacheKey = 'page_block_' + pageId
let data = await getDataFromCache(cacheKey)
if (data && data.pageIds?.length > 0) {
console.debug('[API<<--缓存]', `from:${from}`, `root-page-id:${pageId}`)
return data
export async function getSiteDataByPageId({ pageId, from }) {
// 获取NOTION原始数据此接支持mem缓存。
const pageRecordMap = await getPage(pageId, from)
// 将Notion数据按规则转成站点数据
const data = await converNotionToSiteDate(pageId, from, pageRecordMap)
return data
}
/**
* 获取公告
*/
async function getNotice(post) {
if (!post) {
return null
}
post.blockMap = await getPage(post.id, 'data-notice')
return post
}
/**
* 空的默认数据
* @param {*} pageId
* @returns
*/
const EmptyData = pageId => {
const empty = {
notice: null,
siteInfo: getSiteInfo({}),
allPages: [
{
id: 1,
title: `无法获取Notion数据请检查Notion_ID \n 当前 ${pageId}`,
summary:
'访问文档获取帮助→ https://tangly1024.com/article/vercel-deploy-notion-next',
status: 'Published',
type: 'Post',
slug: '13a171332816461db29d50e9f575b00d',
publishDay: '2024-11-13',
pageCoverThumbnail: BLOG.HOME_BANNER_IMAGE,
date: {
start_date: '2023-04-24',
lastEditedDay: '2023-04-24',
tagItems: []
}
}
],
allNavPages: [],
collection: [],
collectionQuery: {},
collectionId: null,
collectionView: {},
viewIds: [],
block: {},
schema: {},
tagOptions: [],
categoryOptions: [],
rawMetadata: {},
customNav: [],
customMenu: [],
postCount: 1,
pageIds: [],
latestPosts: []
}
return empty
}
/**
* 将Notion数据转站点数据
* 这里统一对数据格式化
* @returns {Promise<JSX.Element|null|*>}
*/
async function converNotionToSiteDate(pageId, from, pageRecordMap) {
if (!pageRecordMap) {
console.error('can`t get Notion Data ; Which id is: ', pageId)
return {}
}
pageId = idToUuid(pageId)
let block = pageRecordMap.block || {}
const rawMetadata = block[pageId]?.value
// Check Type Page-Database和Inline-Database
if (
rawMetadata?.type !== 'collection_view_page' &&
rawMetadata?.type !== 'collection_view'
) {
console.error(`pageId "${pageId}" is not a database`)
return EmptyData(pageId)
}
const collection = Object.values(pageRecordMap.collection)[0]?.value || {}
const collectionId = rawMetadata?.collection_id
const collectionQuery = pageRecordMap.collection_query
const collectionView = pageRecordMap.collection_view
const schema = collection?.schema
const viewIds = rawMetadata?.view_ids
const collectionData = []
const pageIds = getAllPageIds(
collectionQuery,
collectionId,
collectionView,
viewIds
)
if (pageIds?.length === 0) {
console.error(
'获取到的文章列表为空请检查notion模板',
collectionQuery,
collection,
collectionView,
viewIds,
pageRecordMap
)
} else {
// 从接口读取
data = await getDataBaseInfoByNotionAPI({ pageId, from })
// 存入缓存
if (data) {
await setDataToCache(cacheKey, data)
// console.log('有效Page数量', pageIds?.length)
}
// 抓取主数据库最多抓取1000个blocks溢出的数block这里统一抓取一遍
const blockIdsNeedFetch = []
for (let i = 0; i < pageIds.length; i++) {
const id = pageIds[i]
const value = block[id]?.value
if (!value) {
blockIdsNeedFetch.push(id)
}
}
const fetchedBlocks = await fetchInBatches(blockIdsNeedFetch)
block = Object.assign({}, block, fetchedBlocks)
// 获取每篇文章基础数据
for (let i = 0; i < pageIds.length; i++) {
const id = pageIds[i]
const value = block[id]?.value || fetchedBlocks[id]?.value
const properties =
(await getPageProperties(
id,
value,
schema,
null,
getTagOptions(schema)
)) || null
if (properties) {
collectionData.push(properties)
}
}
// 返回给前端的数据做处理
return data
// 站点配置优先读取配置表格否则读取blog.config.js 文件
const NOTION_CONFIG = (await getConfigMapFromConfigPage(collectionData)) || {}
// 处理每一条数据的字段
collectionData.forEach(function (element) {
adjustPageProperties(element, NOTION_CONFIG)
})
// 站点基础信息
const siteInfo = getSiteInfo({ collection, block, pageId })
// 文章计数
let postCount = 0
// 查找所有的Post和Page
const allPages = collectionData.filter(post => {
if (post?.type === 'Post' && post.status === 'Published') {
postCount++
}
return (
post &&
post?.slug &&
// !post?.slug?.startsWith('http') &&
(post?.status === 'Invisible' || post?.status === 'Published')
)
})
// Sort by date
if (siteConfig('POSTS_SORT_BY', '', NOTION_CONFIG) === 'date') {
allPages.sort((a, b) => {
return b?.publishDate - a?.publishDate
})
}
const notice = await getNotice(
collectionData.filter(post => {
return (
post &&
post?.type &&
post?.type === 'Notice' &&
post.status === 'Published'
)
})?.[0]
)
// 所有分类
const categoryOptions = getAllCategories({
allPages,
categoryOptions: getCategoryOptions(schema)
})
// 所有标签
const tagOptions = getAllTags({
allPages,
tagOptions: getTagOptions(schema),
NOTION_CONFIG
})
// 旧的菜单
const customNav = getCustomNav({
allPages: collectionData.filter(
post => post?.type === 'Page' && post.status === 'Published'
)
})
// 新的菜单
const customMenu = await getCustomMenu({ collectionData, NOTION_CONFIG })
const latestPosts = getLatestPosts({ allPages, from, latestPostCount: 6 })
const allNavPages = getNavPages({ allPages })
return {
NOTION_CONFIG,
notice,
siteInfo,
allPages,
allNavPages,
collection,
collectionQuery,
collectionId,
collectionView,
viewIds,
block,
schema,
tagOptions,
categoryOptions,
rawMetadata,
customNav,
customMenu,
postCount,
pageIds,
latestPosts
}
}
/**
@@ -404,228 +620,3 @@ export function getNavPages({ allPages }) {
ext: item.ext || {}
}))
}
/**
* 获取公告
*/
async function getNotice(post) {
if (!post) {
return null
}
post.blockMap = await getPage(post.id, 'data-notice')
return post
}
// 没有数据时返回
const EmptyData = pageId => {
const empty = {
notice: null,
siteInfo: getSiteInfo({}),
allPages: [
{
id: 1,
title: `无法获取Notion数据请检查Notion_ID \n 当前 ${pageId}`,
summary:
'访问文档获取帮助→ https://tangly1024.com/article/vercel-deploy-notion-next',
status: 'Published',
type: 'Post',
slug: '13a171332816461db29d50e9f575b00d',
pageCoverThumbnail: BLOG.HOME_BANNER_IMAGE,
date: {
start_date: '2023-04-24',
lastEditedDay: '2023-04-24',
tagItems: []
}
}
],
allNavPages: [],
collection: [],
collectionQuery: {},
collectionId: null,
collectionView: {},
viewIds: [],
block: {},
schema: {},
tagOptions: [],
categoryOptions: [],
rawMetadata: {},
customNav: [],
customMenu: [],
postCount: 1,
pageIds: [],
latestPosts: []
}
return empty
}
/**
* 调用NotionAPI获取Page数据
* @returns {Promise<JSX.Element|null|*>}
*/
async function getDataBaseInfoByNotionAPI({ pageId, from }) {
console.log('[Fetching Data]', pageId, from)
const pageRecordMap = await getPage(pageId, from)
if (!pageRecordMap) {
console.error('can`t get Notion Data ; Which id is: ', pageId)
return {}
}
pageId = idToUuid(pageId)
let block = pageRecordMap.block || {}
const rawMetadata = block[pageId]?.value
// Check Type Page-Database和Inline-Database
if (
rawMetadata?.type !== 'collection_view_page' &&
rawMetadata?.type !== 'collection_view'
) {
console.error(`pageId "${pageId}" is not a database`)
return EmptyData(pageId)
}
const collection = Object.values(pageRecordMap.collection)[0]?.value || {}
const collectionId = rawMetadata?.collection_id
const collectionQuery = pageRecordMap.collection_query
const collectionView = pageRecordMap.collection_view
const schema = collection?.schema
const viewIds = rawMetadata?.view_ids
const collectionData = []
const pageIds = getAllPageIds(
collectionQuery,
collectionId,
collectionView,
viewIds
)
if (pageIds?.length === 0) {
console.error(
'获取到的文章列表为空请检查notion模板',
collectionQuery,
collection,
collectionView,
viewIds,
pageRecordMap
)
} else {
// console.log('有效Page数量', pageIds?.length)
}
// 抓取主数据库最多抓取1000个blocks溢出的数block这里统一抓取一遍
const blockIdsNeedFetch = []
for (let i = 0; i < pageIds.length; i++) {
const id = pageIds[i]
const value = block[id]?.value
if (!value) {
blockIdsNeedFetch.push(id)
}
}
const fetchedBlocks = await fetchInBatches(blockIdsNeedFetch)
block = Object.assign({}, block, fetchedBlocks)
// 获取每篇文章基础数据
for (let i = 0; i < pageIds.length; i++) {
const id = pageIds[i]
const value = block[id]?.value || fetchedBlocks[id]?.value
const properties =
(await getPageProperties(
id,
value,
schema,
null,
getTagOptions(schema)
)) || null
if (properties) {
collectionData.push(properties)
}
}
// 站点配置优先读取配置表格否则读取blog.config.js 文件
const NOTION_CONFIG = (await getConfigMapFromConfigPage(collectionData)) || {}
// 处理每一条数据的字段
collectionData.forEach(function (element) {
adjustPageProperties(element, NOTION_CONFIG)
})
// 站点基础信息
const siteInfo = getSiteInfo({ collection, block, pageId })
// 文章计数
let postCount = 0
// 查找所有的Post和Page
const allPages = collectionData.filter(post => {
if (post?.type === 'Post' && post.status === 'Published') {
postCount++
}
return (
post &&
post?.slug &&
// !post?.slug?.startsWith('http') &&
(post?.status === 'Invisible' || post?.status === 'Published')
)
})
// Sort by date
if (siteConfig('POSTS_SORT_BY', '', NOTION_CONFIG) === 'date') {
allPages.sort((a, b) => {
return b?.publishDate - a?.publishDate
})
}
const notice = await getNotice(
collectionData.filter(post => {
return (
post &&
post?.type &&
post?.type === 'Notice' &&
post.status === 'Published'
)
})?.[0]
)
// 所有分类
const categoryOptions = getAllCategories({
allPages,
categoryOptions: getCategoryOptions(schema)
})
// 所有标签
const tagOptions = getAllTags({
allPages,
tagOptions: getTagOptions(schema),
NOTION_CONFIG
})
// 旧的菜单
const customNav = getCustomNav({
allPages: collectionData.filter(
post => post?.type === 'Page' && post.status === 'Published'
)
})
// 新的菜单
const customMenu = await getCustomMenu({ collectionData, NOTION_CONFIG })
const latestPosts = getLatestPosts({ allPages, from, latestPostCount: 6 })
const allNavPages = getNavPages({ allPages })
return {
NOTION_CONFIG,
notice,
siteInfo,
allPages,
allNavPages,
collection,
collectionQuery,
collectionId,
collectionView,
viewIds,
block,
schema,
tagOptions,
categoryOptions,
rawMetadata,
customNav,
customMenu,
postCount,
pageIds,
latestPosts
}
}

View File

@@ -157,7 +157,8 @@ export async function getConfigMapFromConfigPage(allPages) {
// 只导入生效的配置
if (config.enable) {
// console.log('[Notion配置]', config.key, config.value)
notionConfig[config.key] = config.value
notionConfig[config.key] = config.value || ''
// 配置不能是undefined至少是空字符串
}
}
}

View File

@@ -180,9 +180,7 @@ export function adjustPageProperties(properties, NOTION_CONFIG) {
// 1.按照用户配置的URL_PREFIX 转换一下slug
// 2.为文章添加一个href字段存储最终调整的路径
if (properties.type === 'Post') {
if (siteConfig('POST_URL_PREFIX', '', NOTION_CONFIG)) {
properties.slug = generateCustomizeSlug(properties, NOTION_CONFIG)
}
properties.slug = generateCustomizeSlug(properties, NOTION_CONFIG)
properties.href = properties.slug ?? properties.id
} else if (properties.type === 'Page') {
properties.href = properties.slug ?? properties.id
@@ -245,11 +243,9 @@ function generateCustomizeSlug(postProperties, NOTION_CONFIG) {
return postProperties.slug
}
let fullPrefix = ''
const allSlugPatterns = siteConfig(
'POST_URL_PREFIX',
'',
NOTION_CONFIG
).split('/')
const allSlugPatterns =
NOTION_CONFIG?.POST_URL_PREFIX ||
siteConfig('POST_URL_PREFIX', '', NOTION_CONFIG).split('/')
const POST_URL_PREFIX_MAPPING_CATEGORY = siteConfig(
'POST_URL_PREFIX_MAPPING_CATEGORY',

View File

@@ -10,19 +10,20 @@ import { deepClone, delay } from '../utils'
* @param {*} slice
* @returns
*/
export async function getPage(id, from, slice) {
const cacheKey = 'page_block_' + id
export async function getPage(id, from = null, slice) {
const cacheKey = `page_block_${id}`
let pageBlock = await getDataFromCache(cacheKey)
if (pageBlock) {
// console.log('[API<<--缓存]', `from:${from}`, cacheKey)
return filterPostBlocks(id, pageBlock, slice)
// console.debug('[API<<--缓存]', `from:${from}`, cacheKey)
return convertNotionBlocksToPost(id, pageBlock, slice)
}
// 抓取最新数据
pageBlock = await getPageWithRetry(id, from)
if (pageBlock) {
await setDataToCache(cacheKey, pageBlock)
return filterPostBlocks(id, pageBlock, slice)
return convertNotionBlocksToPost(id, pageBlock, slice)
}
return pageBlock
}
@@ -69,7 +70,7 @@ export async function getPageWithRetry(id, from, retryAttempts = 3) {
}
/**
* 获取到的页面BLOCK特殊处理
* Notion页面BLOCK格式化处理
* 1.删除冗余字段
* 2.比如文件、视频、音频、url格式化
* 3.代码块等元素兼容
@@ -78,72 +79,72 @@ export async function getPageWithRetry(id, from, retryAttempts = 3) {
* @param {*} slice 截取数量
* @returns
*/
function filterPostBlocks(id, blockMap, slice) {
function convertNotionBlocksToPost(id, blockMap, slice) {
const clonePageBlock = deepClone(blockMap)
let count = 0
const blocksToProcess = Object.keys(clonePageBlock?.block || {})
// 循环遍历文档的每个block
for (let i = 0; i < blocksToProcess.length; i++) {
const blockId = blocksToProcess[i]
const b = clonePageBlock?.block[blockId]
if (slice && slice > 0 && count > slice) {
delete clonePageBlock?.block[blockId]
continue
}
// 当BlockId等于PageId时移除
if (b?.value?.id === id) {
// 此block含有敏感信息
delete b?.value?.properties
continue
}
count++
if (b?.value?.type === 'sync_block' && b?.value?.children) {
const childBlocks = b.value.children
// 移除同步块
delete clonePageBlock.block[blockId]
// 用子块替代同步块
childBlocks.forEach((childBlock, index) => {
const newBlockId = `${blockId}_child_${index}`
clonePageBlock.block[newBlockId] = childBlock
blocksToProcess.splice(i + index + 1, 0, newBlockId)
})
// 重新处理新加入的子块
i--
continue
}
// 处理 c++、c#、汇编等语言名字映射
if (b?.value?.type === 'code') {
if (b?.value?.properties?.language?.[0][0] === 'C++') {
b.value.properties.language[0][0] = 'cpp'
const blockId = blocksToProcess[i]
const b = clonePageBlock?.block[blockId]
if (slice && slice > 0 && count > slice) {
delete clonePageBlock?.block[blockId]
continue
}
if (b?.value?.properties?.language?.[0][0] === 'C#') {
b.value.properties.language[0][0] = 'csharp'
// 当BlockId等于PageId时移除
if (b?.value?.id === id) {
// 此block含有敏感信息
delete b?.value?.properties
continue
}
if (b?.value?.properties?.language?.[0][0] === 'Assembly') {
b.value.properties.language[0][0] = 'asm6502'
count++
if (b?.value?.type === 'sync_block' && b?.value?.children) {
const childBlocks = b.value.children
// 移除同步块
delete clonePageBlock.block[blockId]
// 用子块替代同步块
childBlocks.forEach((childBlock, index) => {
const newBlockId = `${blockId}_child_${index}`
clonePageBlock.block[newBlockId] = childBlock
blocksToProcess.splice(i + index + 1, 0, newBlockId)
})
// 重新处理新加入的子块
i--
continue
}
// 处理 c++、c#、汇编等语言名字映射
if (b?.value?.type === 'code') {
if (b?.value?.properties?.language?.[0][0] === 'C++') {
b.value.properties.language[0][0] = 'cpp'
}
if (b?.value?.properties?.language?.[0][0] === 'C#') {
b.value.properties.language[0][0] = 'csharp'
}
if (b?.value?.properties?.language?.[0][0] === 'Assembly') {
b.value.properties.language[0][0] = 'asm6502'
}
}
// 如果是文件或嵌入式PDF需要重新加密签名
if (
(b?.value?.type === 'file' ||
b?.value?.type === 'pdf' ||
b?.value?.type === 'video' ||
b?.value?.type === 'audio') &&
b?.value?.properties?.source?.[0][0] &&
b?.value?.properties?.source?.[0][0].indexOf('amazonaws.com') > 0
) {
const oldUrl = b?.value?.properties?.source?.[0][0]
const newUrl = `https://notion.so/signed/${encodeURIComponent(oldUrl)}?table=block&id=${b?.value?.id}`
b.value.properties.source[0][0] = newUrl
}
}
// 如果是文件或嵌入式PDF需要重新加密签名
if (
(b?.value?.type === 'file' ||
b?.value?.type === 'pdf' ||
b?.value?.type === 'video' ||
b?.value?.type === 'audio') &&
b?.value?.properties?.source?.[0][0] &&
b?.value?.properties?.source?.[0][0].indexOf('amazonaws.com') > 0
) {
const oldUrl = b?.value?.properties?.source?.[0][0]
const newUrl = `https://notion.so/signed/${encodeURIComponent(oldUrl)}?table=block&id=${b?.value?.id}`
b.value.properties.source[0][0] = newUrl
}
}
// 去掉不用的字段
if (id === BLOG.NOTION_PAGE_ID) {

View File

@@ -1,29 +1,34 @@
import fs from 'fs'
import BLOG from '@/blog.config'
import fs from 'fs'
export async function generateSitemapXml({ allPages }) {
const urls = [{
loc: `${BLOG.LINK}`,
lastmod: new Date().toISOString().split('T')[0],
changefreq: 'daily'
}, {
loc: `${BLOG.LINK}/archive`,
lastmod: new Date().toISOString().split('T')[0],
changefreq: 'daily'
}, {
loc: `${BLOG.LINK}/category`,
lastmod: new Date().toISOString().split('T')[0],
changefreq: 'daily'
}, {
loc: `${BLOG.LINK}/tag`,
lastmod: new Date().toISOString().split('T')[0],
changefreq: 'daily'
}]
const urls = [
{
loc: `${BLOG.LINK}`,
lastmod: new Date().toISOString().split('T')[0],
changefreq: 'daily'
},
{
loc: `${BLOG.LINK}/archive`,
lastmod: new Date().toISOString().split('T')[0],
changefreq: 'daily'
},
{
loc: `${BLOG.LINK}/category`,
lastmod: new Date().toISOString().split('T')[0],
changefreq: 'daily'
},
{
loc: `${BLOG.LINK}/tag`,
lastmod: new Date().toISOString().split('T')[0],
changefreq: 'daily'
}
]
// 循环页面生成
allPages?.forEach(post => {
const slugWithoutLeadingSlash = post?.slug?.startsWith('/') ? post?.slug?.slice(1) : post.slug
const slugWithoutLeadingSlash = post?.slug?.startsWith('/')
? post?.slug?.slice(1)
: post.slug
urls.push({
loc: `${BLOG.LINK}/${slugWithoutLeadingSlash}`,
lastmod: new Date(post?.publishDay).toISOString().split('T')[0],