feat: lots of refactors and improvements

lib/config.ts

@@ -36,6 +36,9 @@ export const pageUrlAdditions = cleanPageUrlMap(
   'pageUrlAdditions'
 )
 
+export const isDev =
+  process.env.NODE_ENV === 'development' || !process.env.NODE_ENV
+
 // general site config
 export const name: string = getSiteConfig('name')
 export const author: string = getSiteConfig('author')
@@ -70,24 +73,20 @@ export const defaultPageCoverPosition: number = getSiteConfig(
   0.5
 )
 
-// Optional utteranc.es comments via GitHub issue comments
-export const utterancesGitHubRepo: string | null = getSiteConfig(
-  'utterancesGitHubRepo',
-  null
-)
-
 // Optional image CDN host to proxy all image requests through
 export const imageCDNHost: string | null = getSiteConfig('imageCDNHost', null)
 
 // Optional whether or not to enable support for LQIP preview images
-// (requires a Google Firebase collection)
 export const isPreviewImageSupportEnabled: boolean = getSiteConfig(
   'isPreviewImageSupportEnabled',
   false
 )
 
-export const isDev =
-  process.env.NODE_ENV === 'development' || !process.env.NODE_ENV
+// Optional whether or not to enable support for LQIP preview images
+export const isTweetEmbedSupportEnabled: boolean = getSiteConfig(
+  'isTweetEmbedSupportEnabled',
+  true
+)
 
 // where it all starts -- the site's root Notion page
 export const includeNotionIdInUrls: boolean = getSiteConfig(
@@ -105,7 +104,6 @@ export const host = isDev ? `http://localhost:${port}` : `https://${domain}`
 export const apiBaseUrl = `${host}/api`
 
 export const api = {
-  createPreviewImage: `${apiBaseUrl}/create-preview-image`,
   searchNotion: `${apiBaseUrl}/search-notion`
 }
 
@@ -119,46 +117,6 @@ export const fathomConfig = fathomId
     }
   : undefined
 
-const defaultEnvValueForPreviewImageSupport =
-  isPreviewImageSupportEnabled && isServer ? undefined : null
-
-export const googleProjectId = getEnv(
-  'GCLOUD_PROJECT',
-  defaultEnvValueForPreviewImageSupport
-)
-
-export const googleApplicationCredentials = getGoogleApplicationCredentials()
-
-export const firebaseCollectionImages = getEnv(
-  'FIREBASE_COLLECTION_IMAGES',
-  defaultEnvValueForPreviewImageSupport
-)
-
-// this hack is necessary because vercel doesn't support secret files so we need to encode our google
-// credentials a base64-encoded string of the JSON-ified content
-function getGoogleApplicationCredentials() {
-  if (!isPreviewImageSupportEnabled || !isServer) {
-    return null
-  }
-
-  try {
-    const googleApplicationCredentialsBase64 = getEnv(
-      'GOOGLE_APPLICATION_CREDENTIALS',
-      defaultEnvValueForPreviewImageSupport
-    )
-
-    return JSON.parse(
-      Buffer.from(googleApplicationCredentialsBase64, 'base64').toString()
-    )
-  } catch (err) {
-    console.error(
-      'Firebase config error: invalid "GOOGLE_APPLICATION_CREDENTIALS" should be base64-encoded JSON\n'
-    )
-
-    throw err
-  }
-}
-
 function cleanPageUrlMap(
   pageUrlMap: PageUrlOverridesMap,
   label: string

lib/db.ts

@@ -1,14 +0,0 @@
-import * as firestore from '@google-cloud/firestore'
-import * as config from './config'
-
-export let db: firestore.Firestore = null
-export let images: firestore.CollectionReference = null
-
-if (config.isPreviewImageSupportEnabled) {
-  db = new firestore.Firestore({
-    projectId: config.googleProjectId,
-    credentials: config.googleApplicationCredentials
-  })
-
-  images = db.collection(config.firebaseCollectionImages)
-}

lib/get-preview-images.ts

@@ -1,58 +0,0 @@
-import crypto from 'crypto'
-import got from 'got'
-import pMap from 'p-map'
-
-import { api, isPreviewImageSupportEnabled } from './config'
-import * as types from './types'
-import * as db from './db'
-
-function sha256(input: Buffer | string) {
-  const buffer = Buffer.isBuffer(input) ? input : Buffer.from(input)
-  return crypto.createHash('sha256').update(buffer).digest('hex')
-}
-
-export async function getPreviewImages(
-  images: string[]
-): Promise<types.PreviewImageMap> {
-  if (!isPreviewImageSupportEnabled) {
-    return {}
-  }
-
-  const imageDocRefs = images.map((url) => {
-    const id = sha256(url)
-    return db.images.doc(id)
-  })
-
-  if (!imageDocRefs.length) {
-    return {}
-  }
-
-  const imageDocs = await db.db.getAll(...imageDocRefs)
-  const results = await pMap(imageDocs, async (model, index) => {
-    if (model.exists) {
-      return model.data() as types.PreviewImage
-    } else {
-      const json = {
-        url: images[index],
-        id: model.id
-      }
-      console.log('createPreviewImage server-side', json)
-
-      // TODO: should we fire and forget here to speed up builds?
-      return got
-        .post(api.createPreviewImage, { json })
-        .json() as Promise<types.PreviewImage>
-    }
-  })
-
-  return results
-    .filter(Boolean)
-    .filter((image) => !image.error)
-    .reduce(
-      (acc, result) => ({
-        ...acc,
-        [result.url]: result
-      }),
-      {}
-    )
-}

lib/map-image-url.ts

@@ -48,7 +48,6 @@ export const mapImageUrl = (imageUrl: string) => {
   }
 
   if (imageCDNHost) {
-    // Our proxy uses Cloudflare's global CDN to cache these image assets
     return `${imageCDNHost}/${encodeURIComponent(imageUrl)}`
   } else {
     return imageUrl

lib/notion.ts

@@ -1,9 +1,12 @@
 import { NotionAPI } from 'notion-client'
 import { ExtendedRecordMap, SearchParams, SearchResults } from 'notion-types'
-import { getPreviewImages } from './get-preview-images'
-import { mapNotionImageUrl } from './map-image-url'
-import { fetchTweetAst } from 'static-tweets'
-import pMap from 'p-map'
+
+import { getPreviewImageMap } from './preview-images'
+import { getTweetAstMap } from './tweet-embeds'
+import {
+  isPreviewImageSupportEnabled,
+  isTweetEmbedSupportEnabled
+} from './config'
 
 export const notion = new NotionAPI({
   apiBaseUrl: process.env.NOTION_API_BASE_URL
@@ -11,92 +14,16 @@ export const notion = new NotionAPI({
 
 export async function getPage(pageId: string): Promise<ExtendedRecordMap> {
   const recordMap = await notion.getPage(pageId)
-  const blockIds = Object.keys(recordMap.block)
 
-  const imageUrls: string[] = blockIds
-    .map((blockId) => {
-      const block = recordMap.block[blockId]?.value
+  if (isPreviewImageSupportEnabled) {
+    const previewImageMap = await getPreviewImageMap(recordMap)
+    ;(recordMap as any).preview_images = previewImageMap
+  }
 
-      if (block) {
-        if (block.type === 'image') {
-          const source = block.properties?.source?.[0]?.[0]
-
-          if (source) {
-            return {
-              block,
-              url: source
-            }
-          }
-        }
-
-        if ((block.format as any)?.page_cover) {
-          const source = (block.format as any).page_cover
-
-          return {
-            block,
-            url: source
-          }
-        }
-      }
-
-      return null
-    })
-    .filter(Boolean)
-    .map(({ block, url }) => mapNotionImageUrl(url, block))
-    .filter(Boolean)
-
-  const urls = Array.from(new Set(imageUrls))
-  const previewImageMap = await getPreviewImages(urls)
-  ;(recordMap as any).preview_images = previewImageMap
-
-  const tweetIds: string[] = blockIds
-    .map((blockId) => {
-      const block = recordMap.block[blockId]?.value
-
-      if (block) {
-        if (block.type === 'tweet') {
-          const src = block.properties?.source?.[0]?.[0]
-
-          if (src) {
-            const id = src.split('?')[0].split('/').pop()
-            if (id) return id
-          }
-        }
-      }
-
-      return null
-    })
-    .filter(Boolean)
-
-  const tweetAsts = await pMap(
-    tweetIds,
-    async (tweetId) => {
-      try {
-        return {
-          tweetId,
-          tweetAst: await fetchTweetAst(tweetId)
-        }
-      } catch (err) {
-        console.error('error fetching tweet info', tweetId, err)
-      }
-    },
-    {
-      concurrency: 4
-    }
-  )
-
-  const tweetAstMap = tweetAsts.reduce((acc, { tweetId, tweetAst }) => {
-    if (tweetAst) {
-      return {
-        ...acc,
-        [tweetId]: tweetAst
-      }
-    } else {
-      return acc
-    }
-  }, {})
-
-  ;(recordMap as any).tweetAstMap = tweetAstMap
+  if (isTweetEmbedSupportEnabled) {
+    const tweetAstMap = await getTweetAstMap(recordMap)
+    ;(recordMap as any).tweetAstMap = tweetAstMap
+  }
 
   return recordMap
 }

lib/preview-images.ts

@@ -0,0 +1,79 @@
+import got from 'got'
+import lqip from 'lqip-modern'
+import pMap from 'p-map'
+import pMemoize from 'p-memoize'
+
+import { ExtendedRecordMap, PreviewImage, PreviewImageMap } from 'notion-types'
+import { mapNotionImageUrl } from './map-image-url'
+
+// NOTE: this is just an example of how to pre-compute preview images.
+// Depending on how many images you're working with, this can potentially be
+// very expensive to recompute, so in production we recommend that you cache
+// the preview image results in a key-value database of your choosing.
+// If you're not sure where to start, check out https://github.com/jaredwray/keyv
+
+export async function getPreviewImageMap(
+  recordMap: ExtendedRecordMap
+): Promise<PreviewImageMap> {
+  const blockIds = Object.keys(recordMap.block)
+  const imageUrls: string[] = blockIds
+    .map((blockId) => {
+      const block = recordMap.block[blockId]?.value
+
+      if (block) {
+        if (block.type === 'image') {
+          const signedUrl = recordMap.signed_urls?.[block.id]
+          const source = signedUrl || block.properties?.source?.[0]?.[0]
+
+          if (source) {
+            return {
+              block,
+              url: source
+            }
+          }
+        }
+
+        if ((block.format as any)?.page_cover) {
+          const source = (block.format as any).page_cover
+
+          return {
+            block,
+            url: source
+          }
+        }
+      }
+
+      return null
+    })
+    .filter(Boolean)
+    .map(({ block, url }) => mapNotionImageUrl(url, block))
+    .filter(Boolean)
+
+  const urls = Array.from(new Set(imageUrls))
+  const previewImagesMap = Object.fromEntries(
+    await pMap(urls, async (url) => [url, await getPreviewImage(url)], {
+      concurrency: 8
+    })
+  )
+
+  return previewImagesMap
+}
+
+async function createPreviewImage(url: string): Promise<PreviewImage | null> {
+  try {
+    const { body } = await got(url, { responseType: 'buffer' })
+    const result = await lqip(body)
+    console.log('lqip', result.metadata)
+
+    return {
+      originalWidth: result.metadata.originalWidth,
+      originalHeight: result.metadata.originalHeight,
+      dataURIBase64: result.metadata.dataURIBase64
+    }
+  } catch (err) {
+    console.warn('error creating preview image', url, err)
+    return null
+  }
+}
+
+export const getPreviewImage = pMemoize(createPreviewImage)

lib/tweet-embeds.ts

@@ -0,0 +1,55 @@
+import { ExtendedRecordMap } from 'notion-types'
+import { fetchTweetAst } from 'static-tweets'
+import pMap from 'p-map'
+
+export async function getTweetAstMap(recordMap: ExtendedRecordMap) {
+  const blockIds = Object.keys(recordMap.block)
+  const tweetIds: string[] = blockIds
+    .map((blockId) => {
+      const block = recordMap.block[blockId]?.value
+
+      if (block) {
+        if (block.type === 'tweet') {
+          const src = block.properties?.source?.[0]?.[0]
+
+          if (src) {
+            const id = src.split('?')[0].split('/').pop()
+            if (id) return id
+          }
+        }
+      }
+
+      return null
+    })
+    .filter(Boolean)
+
+  const tweetAsts = await pMap(
+    tweetIds,
+    async (tweetId) => {
+      try {
+        return {
+          tweetId,
+          tweetAst: await fetchTweetAst(tweetId)
+        }
+      } catch (err) {
+        console.error('error fetching tweet info', tweetId, err)
+      }
+    },
+    {
+      concurrency: 4
+    }
+  )
+
+  const tweetAstMap = tweetAsts.reduce((acc, { tweetId, tweetAst }) => {
+    if (tweetAst) {
+      return {
+        ...acc,
+        [tweetId]: tweetAst
+      }
+    } else {
+      return acc
+    }
+  }, {})
+
+  return tweetAstMap
+}