From 69b631e57aebfed3379ccefd962465a6d6717342 Mon Sep 17 00:00:00 2001 From: Travis Fischer Date: Thu, 21 Apr 2022 20:46:51 -0400 Subject: [PATCH] feat: add rss feed and cleanup some sitemap code --- lib/config.ts | 11 ++- lib/get-site-for-domain.ts | 14 ---- lib/{get-all-pages.ts => get-site-map.ts} | 19 ++++- lib/get-site-maps.ts | 35 ---------- lib/get-sites.ts | 7 -- lib/resolve-notion-page.ts | 22 ++---- lib/types.ts | 14 +--- package.json | 3 +- pages/[pageId].tsx | 22 +++--- pages/api/social-image.tsx | 8 +-- pages/feed.xml.tsx | 84 +++++++++++++++++++++++ pages/robots.txt.tsx | 2 +- pages/sitemap.xml.tsx | 10 +-- yarn.lock | 25 +++++++ 14 files changed, 162 insertions(+), 114 deletions(-) delete mode 100644 lib/get-site-for-domain.ts rename lib/{get-all-pages.ts => get-site-map.ts} (81%) delete mode 100644 lib/get-site-maps.ts delete mode 100644 lib/get-sites.ts create mode 100644 pages/feed.xml.tsx diff --git a/lib/config.ts b/lib/config.ts index 40618fd..68de305 100644 --- a/lib/config.ts +++ b/lib/config.ts @@ -12,7 +12,8 @@ import { NavigationLink } from './site-config' import { PageUrlOverridesInverseMap, PageUrlOverridesMap, - NavigationStyle + NavigationStyle, + Site } from './types' export const rootNotionPageId: string = parsePageId( @@ -132,6 +133,14 @@ export const api = { // ---------------------------------------------------------------------------- +export const site: Site = { + domain, + name, + rootNotionPageId, + rootNotionSpaceId, + description +} + export const fathomId = isDev ? null : process.env.NEXT_PUBLIC_FATHOM_ID export const fathomConfig = fathomId ? { diff --git a/lib/get-site-for-domain.ts b/lib/get-site-for-domain.ts deleted file mode 100644 index 10391fe..0000000 --- a/lib/get-site-for-domain.ts +++ /dev/null @@ -1,14 +0,0 @@ -import * as config from './config' -import * as types from './types' - -export const getSiteForDomain = async ( - domain: string -): Promise => { - return { - domain, - name: config.name, - rootNotionPageId: config.rootNotionPageId, - rootNotionSpaceId: config.rootNotionSpaceId, - description: config.description - } as types.Site -} diff --git a/lib/get-all-pages.ts b/lib/get-site-map.ts similarity index 81% rename from lib/get-all-pages.ts rename to lib/get-site-map.ts index 9b9c77f..1f74fc9 100644 --- a/lib/get-all-pages.ts +++ b/lib/get-site-map.ts @@ -1,18 +1,31 @@ import pMemoize from 'p-memoize' import { getAllPagesInSpace, uuidToId } from 'notion-utils' -import * as types from './types' import { includeNotionIdInUrls } from './config' import { notion } from './notion-api' import { getCanonicalPageId } from './get-canonical-page-id' +import * as config from './config' +import * as types from './types' const uuid = !!includeNotionIdInUrls -export const getAllPages = pMemoize(getAllPagesImpl, { +export async function getSiteMap(): Promise { + const partialSiteMap = await getAllPages( + config.rootNotionPageId, + config.rootNotionSpaceId + ) + + return { + site: config.site, + ...partialSiteMap + } as types.SiteMap +} + +const getAllPages = pMemoize(getAllPagesImpl, { cacheKey: (...args) => JSON.stringify(args) }) -export async function getAllPagesImpl( +async function getAllPagesImpl( rootNotionPageId: string, rootNotionSpaceId: string ): Promise> { diff --git a/lib/get-site-maps.ts b/lib/get-site-maps.ts deleted file mode 100644 index 55173da..0000000 --- a/lib/get-site-maps.ts +++ /dev/null @@ -1,35 +0,0 @@ -import pMap from 'p-map' - -import { getAllPages } from './get-all-pages' -import { getSites } from './get-sites' -import * as types from './types' - -export async function getSiteMaps(): Promise { - const sites = await getSites() - - const siteMaps = await pMap( - sites, - async (site, index) => { - try { - console.log( - 'getSiteMap', - `${index + 1}/${sites.length}`, - `(${(((index + 1) / sites.length) * 100) | 0}%)`, - site - ) - - return { - site, - ...(await getAllPages(site.rootNotionPageId, site.rootNotionSpaceId)) - } as types.SiteMap - } catch (err) { - console.warn('site build error', index, site, err) - } - }, - { - concurrency: 4 - } - ) - - return siteMaps.filter(Boolean) -} diff --git a/lib/get-sites.ts b/lib/get-sites.ts deleted file mode 100644 index e9c8b31..0000000 --- a/lib/get-sites.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { getSiteForDomain } from './get-site-for-domain' -import * as config from './config' -import * as types from './types' - -export async function getSites(): Promise { - return [await getSiteForDomain(config.domain)] -} diff --git a/lib/resolve-notion-page.ts b/lib/resolve-notion-page.ts index e7deda3..b2ded3e 100644 --- a/lib/resolve-notion-page.ts +++ b/lib/resolve-notion-page.ts @@ -2,15 +2,12 @@ import { parsePageId } from 'notion-utils' import { ExtendedRecordMap } from 'notion-types' import * as acl from './acl' -import * as types from './types' -import { pageUrlOverrides, pageUrlAdditions, environment } from './config' +import { pageUrlOverrides, pageUrlAdditions, environment, site } from './config' import { db } from './db' import { getPage } from './notion' -import { getSiteMaps } from './get-site-maps' -import { getSiteForDomain } from './get-site-for-domain' +import { getSiteMap } from './get-site-map' export async function resolveNotionPage(domain: string, rawPageId?: string) { - let site: types.Site let pageId: string let recordMap: ExtendedRecordMap @@ -47,27 +44,19 @@ export async function resolveNotionPage(domain: string, rawPageId?: string) { } if (pageId) { - ;[site, recordMap] = await Promise.all([ - getSiteForDomain(domain), - getPage(pageId) - ]) + recordMap = await getPage(pageId) } else { // handle mapping of user-friendly canonical page paths to Notion page IDs // e.g., /developer-x-entrepreneur versus /71201624b204481f862630ea25ce62fe - const siteMaps = await getSiteMaps() - const siteMap = siteMaps[0] + const siteMap = await getSiteMap() pageId = siteMap?.canonicalPageMap[rawPageId] if (pageId) { // TODO: we're not re-using the page recordMap from siteMaps because it is // cached aggressively - // site = await getSiteForDomain(domain) // recordMap = siteMap.pageMap[pageId] - ;[site, recordMap] = await Promise.all([ - getSiteForDomain(domain), - getPage(pageId) - ]) + recordMap = await getPage(pageId) if (useUriToPageIdCache) { try { @@ -91,7 +80,6 @@ export async function resolveNotionPage(domain: string, rawPageId?: string) { } } } else { - site = await getSiteForDomain(domain) pageId = site.rootNotionPageId console.log(site) diff --git a/lib/types.ts b/lib/types.ts index ffe1868..37b25ba 100644 --- a/lib/types.ts +++ b/lib/types.ts @@ -16,15 +16,7 @@ export interface PageProps { error?: PageError } -export interface Model { - id: string - userId: string - - createdAt: number - updatedAt: number -} - -export interface Site extends Model { +export interface Site { name: string domain: string @@ -40,10 +32,6 @@ export interface Site extends Model { // opengraph metadata description?: string image?: string - - timestamp: Date - - isDisabled: boolean } export interface SiteMap { diff --git a/package.json b/package.json index f2855bc..c351bc5 100644 --- a/package.json +++ b/package.json @@ -50,7 +50,8 @@ "react-dom": "^17.0.2", "react-notion-x": "^6.12.3", "react-tweet-embed": "^2.0.0", - "react-use": "^17.3.2" + "react-use": "^17.3.2", + "rss": "^1.2.2" }, "devDependencies": { "@next/bundle-analyzer": "^12.1.0", diff --git a/pages/[pageId].tsx b/pages/[pageId].tsx index c3497d7..b92d68b 100644 --- a/pages/[pageId].tsx +++ b/pages/[pageId].tsx @@ -1,6 +1,6 @@ import * as React from 'react' import { isDev, domain } from 'lib/config' -import { getSiteMaps } from 'lib/get-site-maps' +import { getSiteMap } from 'lib/get-site-map' import { resolveNotionPage } from 'lib/resolve-notion-page' import { NotionPage } from 'components' @@ -28,22 +28,20 @@ export async function getStaticPaths() { } } - const siteMaps = await getSiteMaps() + const siteMap = await getSiteMap() - const ret = { - paths: siteMaps.flatMap((siteMap) => - Object.keys(siteMap.canonicalPageMap).map((pageId) => ({ - params: { - pageId - } - })) - ), + const staticPaths = { + paths: Object.keys(siteMap.canonicalPageMap).map((pageId) => ({ + params: { + pageId + } + })), // paths: [], fallback: true } - console.log(ret.paths) - return ret + console.log(staticPaths.paths) + return staticPaths } export default function NotionDomainDynamicPage(props) { diff --git a/pages/api/social-image.tsx b/pages/api/social-image.tsx index 6745a0a..d50718b 100644 --- a/pages/api/social-image.tsx +++ b/pages/api/social-image.tsx @@ -11,10 +11,9 @@ import { import { PageBlock } from 'notion-types' import { notion } from 'lib/notion-api' -import { getSiteForDomain } from 'lib/get-site-for-domain' import { mapImageUrl } from 'lib/map-image-url' -import * as config from 'lib/config' import { interRegular } from 'lib/fonts' +import * as config from 'lib/config' /** * Social image generation via headless chrome. @@ -35,7 +34,6 @@ export default withOGImage<'query', 'id'>({ throw new Error('Invalid notion page id') } - const site = await getSiteForDomain(config.domain) const recordMap = await notion.getPage(pageId) const keys = Object.keys(recordMap?.block || {}) @@ -47,7 +45,7 @@ export default withOGImage<'query', 'id'>({ const isBlogPost = block.type === 'page' && block.parent_table === 'collection' - const title = getBlockTitle(block, recordMap) || site.name + const title = getBlockTitle(block, recordMap) || config.name const image = mapImageUrl( getPageProperty('Social Image', block, recordMap) || (block as PageBlock).format?.page_cover || @@ -96,7 +94,7 @@ export default withOGImage<'query', 'id'>({ month: 'long' })} ${dateUpdated.getFullYear()}` : undefined - const detail = date || site.domain + const detail = date || config.domain return ( diff --git a/pages/feed.xml.tsx b/pages/feed.xml.tsx new file mode 100644 index 0000000..59b4b05 --- /dev/null +++ b/pages/feed.xml.tsx @@ -0,0 +1,84 @@ +import RSS from 'rss' +import type { GetServerSideProps } from 'next' +import { getBlockTitle, getPageProperty } from 'notion-utils' + +import * as config from 'lib/config' +import { getSiteMap } from 'lib/get-site-map' +import { getCanonicalPageUrl } from 'lib/map-page-url' +import { getSocialImageUrl } from 'lib/get-social-image-url' + +export const getServerSideProps: GetServerSideProps = async ({ req, res }) => { + if (req.method !== 'GET') { + res.statusCode = 405 + res.setHeader('Content-Type', 'application/json') + res.write(JSON.stringify({ error: 'method not allowed' })) + res.end() + return { props: {} } + } + + const siteMap = await getSiteMap() + const ttlMinutes = 60 + const ttlSeconds = ttlMinutes * 60 + + const feed = new RSS({ + title: config.name, + site_url: config.host, + feed_url: `${config.host}/feed.xml`, + ttl: ttlMinutes + }) + + for (const pagePath of Object.keys(siteMap.canonicalPageMap)) { + const pageId = siteMap.canonicalPageMap[pagePath] + const recordMap = siteMap.pageMap[pageId] + if (!recordMap) continue + + const keys = Object.keys(recordMap?.block || {}) + const block = recordMap?.block?.[keys[0]]?.value + if (!block) continue + + const title = getBlockTitle(block, recordMap) || config.name + const description = + getPageProperty('Description', block, recordMap) || + config.description + const url = getCanonicalPageUrl(config.site, recordMap)(pageId) + const lastUpdatedTime = getPageProperty( + 'Last Updated', + block, + recordMap + ) + const publishedTime = getPageProperty('Published', block, recordMap) + const date = lastUpdatedTime + ? new Date(lastUpdatedTime) + : publishedTime + ? new Date(publishedTime) + : undefined + const socialImageUrl = getSocialImageUrl(pageId) + + feed.item({ + title, + url, + date, + description, + enclosure: socialImageUrl + ? { + url: socialImageUrl, + type: 'image/jpeg' + } + : undefined + }) + } + + const feedText = feed.xml({ indent: true }) + + res.setHeader( + 'Cache-Control', + `public, max-age=${ttlSeconds}, stale-while-revalidate=${ttlSeconds}` + ) + res.setHeader('Content-Type', 'text/xml') + res.write(feedText) + res.end() + + return { props: {} } +} + +export default () => null diff --git a/pages/robots.txt.tsx b/pages/robots.txt.tsx index 93a5940..d82ef1c 100644 --- a/pages/robots.txt.tsx +++ b/pages/robots.txt.tsx @@ -1,4 +1,4 @@ -import { GetServerSideProps } from 'next' +import type { GetServerSideProps } from 'next' import { host } from 'lib/config' export const getServerSideProps: GetServerSideProps = async ({ req, res }) => { diff --git a/pages/sitemap.xml.tsx b/pages/sitemap.xml.tsx index 69684f8..7cd792d 100644 --- a/pages/sitemap.xml.tsx +++ b/pages/sitemap.xml.tsx @@ -1,7 +1,7 @@ -import { GetServerSideProps } from 'next' -import { SiteMap } from 'lib/types' +import type { GetServerSideProps } from 'next' +import type { SiteMap } from 'lib/types' import { host } from 'lib/config' -import { getSiteMaps } from 'lib/get-site-maps' +import { getSiteMap } from 'lib/get-site-map' export const getServerSideProps: GetServerSideProps = async ({ req, res }) => { if (req.method !== 'GET') { @@ -14,7 +14,7 @@ export const getServerSideProps: GetServerSideProps = async ({ req, res }) => { } } - const siteMaps = await getSiteMaps() + const siteMap = await getSiteMap() // cache for up to 8 hours res.setHeader( @@ -22,7 +22,7 @@ export const getServerSideProps: GetServerSideProps = async ({ req, res }) => { 'public, max-age=28800, stale-while-revalidate=28800' ) res.setHeader('Content-Type', 'text/xml') - res.write(createSitemap(siteMaps[0])) + res.write(createSitemap(siteMap)) res.end() return { diff --git a/yarn.lock b/yarn.lock index 5688d3e..57e9cd1 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2269,6 +2269,18 @@ micromatch@^4.0.2, micromatch@^4.0.4: braces "^3.0.2" picomatch "^2.3.1" +mime-db@~1.25.0: + version "1.25.0" + resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.25.0.tgz#c18dbd7c73a5dbf6f44a024dc0d165a1e7b1c392" + integrity sha1-wY29fHOl2/b0SgJNwNFloeexw5I= + +mime-types@2.1.13: + version "2.1.13" + resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.13.tgz#e07aaa9c6c6b9a7ca3012c69003ad25a39e92a88" + integrity sha1-4HqqnGxrmnyjASxpADrSWjnpKog= + dependencies: + mime-db "~1.25.0" + mimic-fn@^4.0.0: version "4.0.0" resolved "https://registry.npmjs.org/mimic-fn/-/mimic-fn-4.0.0.tgz" @@ -3204,6 +3216,14 @@ rrweb-snapshot@^1.1.7: resolved "https://registry.npmjs.org/rrweb-snapshot/-/rrweb-snapshot-1.1.14.tgz" integrity sha512-eP5pirNjP5+GewQfcOQY4uBiDnpqxNRc65yKPW0eSoU1XamDfc4M8oqpXGMyUyvLyxFDB0q0+DChuxxiU2FXBQ== +rss@^1.2.2: + version "1.2.2" + resolved "https://registry.yarnpkg.com/rss/-/rss-1.2.2.tgz#50a1698876138133a74f9a05d2bdc8db8d27a921" + integrity sha1-UKFpiHYTgTOnT5oF0r3I240nqSE= + dependencies: + mime-types "2.1.13" + xml "1.0.1" + rtl-css-js@^1.14.0: version "1.15.0" resolved "https://registry.npmjs.org/rtl-css-js/-/rtl-css-js-1.15.0.tgz" @@ -3851,6 +3871,11 @@ ws@^7.2.3, ws@^7.3.1: resolved "https://registry.npmjs.org/ws/-/ws-7.5.7.tgz" integrity sha512-KMvVuFzpKBuiIXW3E4u3mySRO2/mCHSyZDJQM5NQ9Q9KHWHWh0NHgfbRMLLrceUK5qAL4ytALJbpRMjixFZh8A== +xml@1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/xml/-/xml-1.0.1.tgz#78ba72020029c5bc87b8a81a3cfcd74b4a2fc1e5" + integrity sha1-eLpyAgApxbyHuKgaPPzXS0ovweU= + yauzl@^2.10.0: version "2.10.0" resolved "https://registry.npmjs.org/yauzl/-/yauzl-2.10.0.tgz"