feat: add redis db to persist images; bug fixes

2026-05-13 15:09:47 +00:00 · 2022-03-23 05:18:09 -04:00
parent 5417bb9bbc
commit 40972ecca6
10 changed files with 166 additions and 76 deletions
--- a/.env.example
+++ b/.env.example
@@ -10,5 +10,12 @@
 # Optional (for fathom analytics)
 #NEXT_PUBLIC_FATHOM_ID=

-# Optional (for rendering tweets efficiently)
-TWITTER_ACCESS_TOKEN=
+# Optional (for rendering tweets more efficiently)
+#TWITTER_ACCESS_TOKEN=
+
+# Optional (for persisting preview images to redis)
+# NOTE: if you want to enable redis, only REDIS_HOST and REDIS_PASSWORD are required
+#REDIS_HOST=
+#REDIS_PASSWORD=
+#REDIS_USER='default'
+#REDIS_NAMESPACE='preview-images'
--- a/components/NotionPage.tsx
+++ b/components/NotionPage.tsx
@@ -18,7 +18,7 @@ import { NotionRenderer, Code, Collection, CollectionRow } from 'react-notion-x'
 // utils
 import { getBlockTitle } from 'notion-utils'
 import { mapPageUrl, getCanonicalPageUrl } from 'lib/map-page-url'
-import { mapNotionImageUrl } from 'lib/map-image-url'
+import { mapImageUrl } from 'lib/map-image-url'
 import { getPageDescription } from 'lib/get-page-description'
 import { getPageTweet } from 'lib/get-page-tweet'
 import { searchNotion } from 'lib/search-notion'
@@ -129,7 +129,7 @@ export const NotionPage: React.FC<types.PageProps> = ({
  const showTableOfContents = !!isBlogPost
  const minTableOfContentsItems = 3

-  const socialImage = mapNotionImageUrl(
+  const socialImage = mapImageUrl(
    (block as PageBlock).format?.page_cover || config.defaultPageCover,
    block
  )
@@ -282,7 +282,7 @@ export const NotionPage: React.FC<types.PageProps> = ({
        defaultPageCover={config.defaultPageCover}
        defaultPageCoverPosition={config.defaultPageCoverPosition}
        mapPageUrl={siteMapPageUrl}
-        mapImageUrl={mapNotionImageUrl}
+        mapImageUrl={mapImageUrl}
        searchNotion={searchNotion}
        pageAside={pageAside}
        footer={
--- a/lib/config.ts
+++ b/lib/config.ts
@@ -73,9 +73,6 @@ export const defaultPageCoverPosition: number = getSiteConfig(
  0.5
 )

-// Optional image CDN host to proxy all image requests through
-export const imageCDNHost: string | null = getSiteConfig('imageCDNHost', null)
-
 // Optional whether or not to enable support for LQIP preview images
 export const isPreviewImageSupportEnabled: boolean = getSiteConfig(
  'isPreviewImageSupportEnabled',
@@ -96,6 +93,23 @@ export const includeNotionIdInUrls: boolean = getSiteConfig(

 // ----------------------------------------------------------------------------

+// Optional redis instance for persisting preview images
+// (if you want to enable redis, only REDIS_HOST and REDIS_PASSWORD are required)
+// we recommend that you store these in a local `.env` file
+export const redisHost: string | null = getEnv('REDIS_HOST', null)
+export const redisPassword: string | null = getEnv('REDIS_PASSWORD', null)
+export const redisUser: string = getEnv('REDIS_USER', 'default')
+export const redisUrl = getEnv(
+  'REDIS_URL',
+  `redis://${redisUser}:${redisPassword}@${redisHost}`
+)
+export const redisNamespace: string | null = getEnv(
+  'REDIS_NAMESPACE',
+  'preview-images'
+)
+
+// ----------------------------------------------------------------------------
+
 export const isServer = typeof window === 'undefined'

 export const port = getEnv('PORT', '3000')
--- a/lib/db.ts
+++ b/lib/db.ts
@@ -0,0 +1,14 @@
+import Keyv from 'keyv'
+
+import {
+  isPreviewImageSupportEnabled,
+  redisUrl,
+  redisNamespace
+} from './config'
+
+let db: Keyv
+if (isPreviewImageSupportEnabled) {
+  db = new Keyv(redisUrl, { namespace: redisNamespace || undefined })
+}
+
+export { db }
--- a/lib/map-image-url.ts
+++ b/lib/map-image-url.ts
@@ -1,55 +1,3 @@
-import { Block } from 'notion-types'
-import { imageCDNHost } from './config'
+import { defaultMapImageUrl } from 'react-notion-x'

-export const mapNotionImageUrl = (url: string, block: Block) => {
-  if (!url) {
-    return null
-  }
-
-  if (url.startsWith('data:')) {
-    return url
-  }
-
-  if (imageCDNHost && url.startsWith(imageCDNHost)) {
-    return url
-  }
-
-  // const origUrl = url
-
-  if (url.startsWith('/images')) {
-    url = `https://www.notion.so${url}`
-  }
-
-  // more recent versions of notion don't proxy unsplash images
-  if (!url.startsWith('https://images.unsplash.com')) {
-    url = `https://www.notion.so${
-      url.startsWith('/image') ? url : `/image/${encodeURIComponent(url)}`
-    }`
-
-    const notionImageUrlV2 = new URL(url)
-    let table = block.parent_table === 'space' ? 'block' : block.parent_table
-    if (table === 'collection') {
-      table = 'block'
-    }
-    notionImageUrlV2.searchParams.set('table', table)
-    notionImageUrlV2.searchParams.set('id', block.id)
-    notionImageUrlV2.searchParams.set('cache', 'v2')
-
-    url = notionImageUrlV2.toString()
-  }
-
-  // console.log({ url, origUrl })
-  return mapImageUrl(url)
-}
-
-export const mapImageUrl = (imageUrl: string) => {
-  if (imageUrl.startsWith('data:')) {
-    return imageUrl
-  }
-
-  if (imageCDNHost) {
-    return `${imageCDNHost}/${encodeURIComponent(imageUrl)}`
-  } else {
-    return imageUrl
-  }
-}
+export const mapImageUrl = defaultMapImageUrl
--- a/lib/preview-images.ts
+++ b/lib/preview-images.ts
@@ -2,9 +2,10 @@ import got from 'got'
 import lqip from 'lqip-modern'
 import pMap from 'p-map'
 import pMemoize from 'p-memoize'
-
 import { ExtendedRecordMap, PreviewImage, PreviewImageMap } from 'notion-types'
-import { mapNotionImageUrl } from './map-image-url'
+
+import { db } from './db'
+import { mapImageUrl } from './map-image-url'

 // NOTE: this is just an example of how to pre-compute preview images.
 // Depending on how many images you're working with, this can potentially be
@@ -46,7 +47,7 @@ export async function getPreviewImageMap(
      return null
    })
    .filter(Boolean)
-    .map(({ block, url }) => mapNotionImageUrl(url, block))
+    .map(({ block, url }) => mapImageUrl(url, block))
    .filter(Boolean)

  const urls = Array.from(new Set(imageUrls))
@@ -60,16 +61,26 @@ export async function getPreviewImageMap(
 }

 async function createPreviewImage(url: string): Promise<PreviewImage | null> {
+  const cacheKey = url
+
  try {
+    const cachedPreviewImage = await db.get(cacheKey)
+    if (cachedPreviewImage) {
+      return cachedPreviewImage
+    }
+
    const { body } = await got(url, { responseType: 'buffer' })
    const result = await lqip(body)
    console.log('lqip', result.metadata)

-    return {
+    const previewImage = {
      originalWidth: result.metadata.originalWidth,
      originalHeight: result.metadata.originalHeight,
      dataURIBase64: result.metadata.dataURIBase64
    }
+
+    await db.set(cacheKey, previewImage)
+    return previewImage
  } catch (err) {
    console.warn('error creating preview image', url, err)
    return null
--- a/package.json
+++ b/package.json
@@ -25,11 +25,13 @@
    "test:prettier": "prettier '**/*.{js,jsx,ts,tsx}' --check"
  },
  "dependencies": {
+    "@keyv/redis": "^2.2.3",
    "classnames": "^2.3.1",
    "date-fns": "^2.25.0",
    "fathom-client": "^3.0.0",
    "got": "^11.8.2",
    "isomorphic-unfetch": "^3.1.0",
+    "keyv": "^4.1.1",
    "lqip-modern": "^1.2.0",
    "next": "^12.1.0",
    "node-fetch": "^2.6.1",
@@ -42,7 +44,7 @@
    "react-body-classname": "^1.3.1",
    "react-dom": "^17.0.2",
    "react-icons": "^4.3.1",
-    "react-notion-x": "^4.19.2",
+    "react-notion-x": "^4.19.6",
    "react-static-tweets": "^0.7.1",
    "react-use": "^17.3.2",
    "static-tweets": "^0.7.1",
--- a/site.config.js
+++ b/site.config.js
@@ -27,6 +27,11 @@ module.exports = {
  defaultPageCover: null,
  defaultPageCoverPosition: 0.5,

+  // whether or not to enable support for LQIP preview images (optional)
+  // NOTE: this requires you to set up an external key-value store and add the
+  // environment variables specified in .env.example
+  isPreviewImageSupportEnabled: true,
+
  // map of notion page IDs to URL paths (optional)
  // any pages defined here will override their default URL paths
  // example:
--- a/styles/notion.css
+++ b/styles/notion.css
@@ -85,7 +85,6 @@

 .notion-collection-card-cover {
  border-radius: 16px;
-  overflow: visible;
  box-shadow: 2px 2px 8px 4px rgba(15, 15, 15, 0.1);
 }

@@ -179,14 +178,21 @@
  padding: 0;
 }

-.notion-page-cover {
-  max-width: 1200px;
+.notion-page-cover-wrapper,
+.notion-page-cover-wrapper span,
+.notion-page-cover-wrapper img {
+  max-width: 1200px !important;
  border-radius: 24px;
+}
+
+.notion-page-cover-wrapper {
  box-shadow: 2px 2px 8px 4px rgba(15, 15, 15, 0.1);
 }

@media only screen and (max-width: 1200px) {
-  .notion-page-cover {
+  .notion-page-cover-wrapper,
+  .notion-page-cover-wrapper span,
+  .notion-page-cover-wrapper img {
    border-radius: 0;
  }
 }
--- a/yarn.lock
+++ b/yarn.lock
@@ -38,6 +38,13 @@
  resolved "https://registry.yarnpkg.com/@humanwhocodes/object-schema/-/object-schema-1.2.1.tgz#b520529ec21d8e5945a1851dfd1c32e94e39ff45"
  integrity sha512-ZnQMnLV4e7hDlUvw8H+U8ASL02SS2Gn6+9Ac3wGGLIe7+je2AeAOxPY+izIPJDfFDb7eDjev0Us8MO1iFRN8hA==

+"@keyv/redis@^2.2.3":
+  version "2.2.3"
+  resolved "https://registry.yarnpkg.com/@keyv/redis/-/redis-2.2.3.tgz#af5b1ea32d847a63ce24012844af7323b3c421a7"
+  integrity sha512-d9Maf1LzT6Ti5hWsVzaWFriFmXrscK1eUl/etNquQgAJxH7Drecbn+uNZXMc6xb78Ju9szy0fD9RAp/G9RzAdg==
+  dependencies:
+    ioredis "^4.28.5"
+
 "@mapbox/rehype-prism@^0.5.0":
  version "0.5.0"
  resolved "https://registry.yarnpkg.com/@mapbox/rehype-prism/-/rehype-prism-0.5.0.tgz#b756308ebf3af8f92a6359cd78010a7770453e85"
@@ -745,6 +752,11 @@ clone-response@^1.0.2:
  dependencies:
    mimic-response "^1.0.0"

+cluster-key-slot@^1.1.0:
+  version "1.1.0"
+  resolved "https://registry.yarnpkg.com/cluster-key-slot/-/cluster-key-slot-1.1.0.tgz#30474b2a981fb12172695833052bc0d01336d10d"
+  integrity sha512-2Nii8p3RwAPiFwsnZvukotvow2rIHM+yQ6ZcBXGHdniadkYGZYiGmkHJIbZPIV9nfv7m/U1IPMVVcAhoWFeklw==
+
 code-point-at@^1.0.0:
  version "1.1.0"
  resolved "https://registry.yarnpkg.com/code-point-at/-/code-point-at-1.1.0.tgz#0d070b4d043a5bea33a2f1a40e2edb3d9a4ccf77"
@@ -915,7 +927,7 @@ debug@^3.1.0:
  dependencies:
    ms "^2.1.1"

-debug@^4.0.0:
+debug@^4.0.0, debug@^4.3.1:
  version "4.3.4"
  resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.4.tgz#1319f6579357f2338d3337d2cdd4914bb5dcc865"
  integrity sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==
@@ -982,6 +994,11 @@ delegates@^1.0.0:
  resolved "https://registry.yarnpkg.com/delegates/-/delegates-1.0.0.tgz#84c6e159b81904fdca59a0ef44cd870d31250f9a"
  integrity sha1-hMbhWbgZBP3KWaDvRM2HDTElD5o=

+denque@^1.1.0:
+  version "1.5.1"
+  resolved "https://registry.yarnpkg.com/denque/-/denque-1.5.1.tgz#07f670e29c9a78f8faecb2566a1e2c11929c5cbf"
+  integrity sha512-XwE+iZ4D6ZUB7mfYRMb5wByE8L74HCn30FBN7sWnXksWc1LO1bPDl67pBR9o/kC4z/xSNAwkMYcGgqDV3BE3Hw==
+
 dequal@2.0.2:
  version "2.0.2"
  resolved "https://registry.yarnpkg.com/dequal/-/dequal-2.0.2.tgz#85ca22025e3a87e65ef75a7a437b35284a7e319d"
@@ -1873,6 +1890,23 @@ invariant@^2.2.4:
  dependencies:
    loose-envify "^1.0.0"

+ioredis@^4.28.5:
+  version "4.28.5"
+  resolved "https://registry.yarnpkg.com/ioredis/-/ioredis-4.28.5.tgz#5c149e6a8d76a7f8fa8a504ffc85b7d5b6797f9f"
+  integrity sha512-3GYo0GJtLqgNXj4YhrisLaNNvWSNwSS2wS4OELGfGxH8I69+XfNdnmV1AyN+ZqMh0i7eX+SWjrwFKDBDgfBC1A==
+  dependencies:
+    cluster-key-slot "^1.1.0"
+    debug "^4.3.1"
+    denque "^1.1.0"
+    lodash.defaults "^4.2.0"
+    lodash.flatten "^4.4.0"
+    lodash.isarguments "^3.1.0"
+    p-map "^2.1.0"
+    redis-commands "1.7.0"
+    redis-errors "^1.2.0"
+    redis-parser "^3.0.0"
+    standard-as-callback "^2.1.0"
+
 is-alphabetical@^1.0.0:
  version "1.0.4"
  resolved "https://registry.yarnpkg.com/is-alphabetical/-/is-alphabetical-1.0.4.tgz#9e7d6b94916be22153745d184c298cbf986a686d"
@@ -2136,6 +2170,13 @@ keyv@^4.0.0:
  dependencies:
    json-buffer "3.0.1"

+keyv@^4.1.1:
+  version "4.1.1"
+  resolved "https://registry.yarnpkg.com/keyv/-/keyv-4.1.1.tgz#02c538bfdbd2a9308cc932d4096f05ae42bfa06a"
+  integrity sha512-tGv1yP6snQVDSM4X6yxrv2zzq/EvpW+oYiUz6aueW1u9CtS8RzUQYxxmFwgZlO2jSgCxQbchhxaqXXp2hnKGpQ==
+  dependencies:
+    json-buffer "3.0.1"
+
 levn@^0.4.1:
  version "0.4.1"
  resolved "https://registry.yarnpkg.com/levn/-/levn-0.4.1.tgz#ae4562c007473b932a6200d403268dd2fffc6ade"
@@ -2163,6 +2204,21 @@ loader-utils@^2.0.0:
    emojis-list "^3.0.0"
    json5 "^2.1.2"

+lodash.defaults@^4.2.0:
+  version "4.2.0"
+  resolved "https://registry.yarnpkg.com/lodash.defaults/-/lodash.defaults-4.2.0.tgz#d09178716ffea4dde9e5fb7b37f6f0802274580c"
+  integrity sha1-0JF4cW/+pN3p5ft7N/bwgCJ0WAw=
+
+lodash.flatten@^4.4.0:
+  version "4.4.0"
+  resolved "https://registry.yarnpkg.com/lodash.flatten/-/lodash.flatten-4.4.0.tgz#f31c22225a9632d2bbf8e4addbef240aa765a61f"
+  integrity sha1-8xwiIlqWMtK7+OSt2+8kCqdlph8=
+
+lodash.isarguments@^3.1.0:
+  version "3.1.0"
+  resolved "https://registry.yarnpkg.com/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz#2f573d85c6a24289ff00663b491c1d338ff3458a"
+  integrity sha1-L1c9hcaiQon/AGY7SRwdM4/zRYo=
+
 lodash.merge@^4.6.2:
  version "4.6.2"
  resolved "https://registry.yarnpkg.com/lodash.merge/-/lodash.merge-4.6.2.tgz#558aa53b43b661e1925a0afdfa36a9a1085fe57a"
@@ -2655,6 +2711,11 @@ p-finally@^1.0.0:
  resolved "https://registry.yarnpkg.com/p-finally/-/p-finally-1.0.0.tgz#3fbcfb15b899a44123b34b6dcc18b724336a2cae"
  integrity sha1-P7z7FbiZpEEjs0ttzBi3JDNqLK4=

+p-map@^2.1.0:
+  version "2.1.0"
+  resolved "https://registry.yarnpkg.com/p-map/-/p-map-2.1.0.tgz#310928feef9c9ecc65b68b17693018a665cea175"
+  integrity sha512-y3b8Kpd8OAN444hxfBbFfj1FY/RjtTd8tzYwhUqNYXx0fXx2iX4maP4Qr6qhIKbQXI02wTLAda4fYUbDagTUFw==
+
 p-map@^4.0.0:
  version "4.0.0"
  resolved "https://registry.yarnpkg.com/p-map/-/p-map-4.0.0.tgz#bb2f95a5eda2ec168ec9274e06a747c3e2904d2b"
@@ -3054,10 +3115,10 @@ react-modal@^3.14.3:
    react-lifecycles-compat "^3.0.0"
    warning "^4.0.3"

-react-notion-x@^4.19.2:
-  version "4.19.2"
-  resolved "https://registry.yarnpkg.com/react-notion-x/-/react-notion-x-4.19.2.tgz#8f3b06c87a425afa9c52a4b5ce84f9087898b775"
-  integrity sha512-bmK92R58z5iPHWaWxb+QbpyMxukOypNHH2DAK9cDpf+WePEXNnqpcPcp1B03Pi3cexlXn5nTBBfdoyAAJVsEkA==
+react-notion-x@^4.19.6:
+  version "4.19.6"
+  resolved "https://registry.yarnpkg.com/react-notion-x/-/react-notion-x-4.19.6.tgz#7922f650e9113f1328557d7bb89151efc09a8e1e"
+  integrity sha512-JmUXnmCGAifAw4jC9zzQ0qquWgNWnGVLuG74j5fMD9bUdpFibS/LWq2v+irz2fBOWLXA/YX2qKrXarSs1msxww==
  dependencies:
    "@matejmazur/react-katex" "^3.1.3"
    date-fns "^2.15.0"
@@ -3191,6 +3252,23 @@ readable-stream@^3.1.1, readable-stream@^3.4.0:
    string_decoder "^1.1.1"
    util-deprecate "^1.0.1"

+redis-commands@1.7.0:
+  version "1.7.0"
+  resolved "https://registry.yarnpkg.com/redis-commands/-/redis-commands-1.7.0.tgz#15a6fea2d58281e27b1cd1acfb4b293e278c3a89"
+  integrity sha512-nJWqw3bTFy21hX/CPKHth6sfhZbdiHP6bTawSgQBlKOVRG7EZkfHbbHwQJnrE4vsQf0CMNE+3gJ4Fmm16vdVlQ==
+
+redis-errors@^1.0.0, redis-errors@^1.2.0:
+  version "1.2.0"
+  resolved "https://registry.yarnpkg.com/redis-errors/-/redis-errors-1.2.0.tgz#eb62d2adb15e4eaf4610c04afe1529384250abad"
+  integrity sha1-62LSrbFeTq9GEMBK/hUpOEJQq60=
+
+redis-parser@^3.0.0:
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/redis-parser/-/redis-parser-3.0.0.tgz#b66d828cdcafe6b4b8a428a7def4c6bcac31c8b4"
+  integrity sha1-tm2CjNyv5rS4pCin3vTGvKwxyLQ=
+  dependencies:
+    redis-errors "^1.0.0"
+
 refractor@^3.0.0:
  version "3.3.1"
  resolved "https://registry.yarnpkg.com/refractor/-/refractor-3.3.1.tgz#ebbc04b427ea81dc25ad333f7f67a0b5f4f0be3a"
@@ -3562,6 +3640,11 @@ stacktrace-js@^2.0.2:
    stack-generator "^2.0.5"
    stacktrace-gps "^3.0.4"

+standard-as-callback@^2.1.0:
+  version "2.1.0"
+  resolved "https://registry.yarnpkg.com/standard-as-callback/-/standard-as-callback-2.1.0.tgz#8953fc05359868a77b5b9739a665c5977bb7df45"
+  integrity sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==
+
 static-tweets@^0.7.1:
  version "0.7.1"
  resolved "https://registry.yarnpkg.com/static-tweets/-/static-tweets-0.7.1.tgz#162258d172f67d9685c3738ab026ef731cdc51e7"