feat: initial webapp structure from notion2site

lib/acl.ts

@@ -0,0 +1,55 @@
+import { PageProps } from './types'
+
+export async function pageAcl({
+  site,
+  recordMap,
+  pageId
+}: PageProps): Promise<PageProps> {
+  if (!site) {
+    return {
+      error: {
+        statusCode: 404,
+        message: 'Unable to resolve notion site'
+      }
+    }
+  }
+
+  if (!recordMap) {
+    return {
+      error: {
+        statusCode: 404,
+        message: `Unable to resolve page for domain "${site.domain}". Notion page "${pageId}" not found.`
+      }
+    }
+  }
+
+  const keys = Object.keys(recordMap.block)
+  const rootKey = keys[0]
+
+  if (!rootKey) {
+    return {
+      error: {
+        statusCode: 404,
+        message: `Unable to resolve page for domain "${site.domain}". Notion page "${pageId}" invalid data.`
+      }
+    }
+  }
+
+  const rootValue = recordMap.block[rootKey]?.value
+  const rootSpaceId = rootValue?.space_id
+
+  if (
+    rootSpaceId &&
+    site.rootNotionSpaceId &&
+    rootSpaceId !== site.rootNotionSpaceId
+  ) {
+    if (process.env.NODE_ENV) {
+      return {
+        error: {
+          statusCode: 404,
+          message: `Notion page "${pageId}" doesn't belong to the Notion workspace owned by "${site.domain}".`
+        }
+      }
+    }
+  }
+}

lib/bootstrap-client.ts

@@ -0,0 +1,13 @@
+export function bootstrap() {
+  console.log(`
+
+████████╗██████╗  █████╗ ███╗   ██╗███████╗██╗████████╗██╗██╗   ██╗███████╗    ██████╗ ███████╗
+╚══██╔══╝██╔══██╗██╔══██╗████╗  ██║██╔════╝██║╚══██╔══╝██║██║   ██║██╔════╝    ██╔══██╗██╔════╝
+   ██║   ██████╔╝███████║██╔██╗ ██║███████╗██║   ██║   ██║██║   ██║█████╗      ██████╔╝███████╗
+   ██║   ██╔══██╗██╔══██║██║╚██╗██║╚════██║██║   ██║   ██║╚██╗ ██╔╝██╔══╝      ██╔══██╗╚════██║
+   ██║   ██║  ██║██║  ██║██║ ╚████║███████║██║   ██║   ██║ ╚████╔╝ ███████╗    ██████╔╝███████║
+   ╚═╝   ╚═╝  ╚═╝╚═╝  ╚═╝╚═╝  ╚═══╝╚══════╝╚═╝   ╚═╝   ╚═╝  ╚═══╝  ╚══════╝    ╚═════╝ ╚══════╝
+                                                                                               
+   This site is built using Notion, Next.js, and https://github.com/NotionX/react-notion-x.
+`)
+}

lib/config.ts

@@ -0,0 +1,21 @@
+/**
+ * All app config that should be available client-side.
+ *
+ * @see env.ts for server-side version.
+ */
+
+import { getEnv } from './get-env'
+
+export const isDev =
+  process.env.NODE_ENV === 'development' || !process.env.NODE_ENV
+
+export const defaultSiteImage = '/social.jpg'
+export const defaultSiteFavicon = '/favicon.ico'
+
+export const fathomId = isDev ? null : getEnv('FATHOM_ID', null)
+
+export const fathomConfig = fathomId
+  ? {
+      excludedDomains: ['localhost', 'localhost:3000']
+    }
+  : undefined

lib/db.ts

@@ -0,0 +1,48 @@
+import * as firestore from '@google-cloud/firestore'
+import * as types from './types'
+import * as config from './env'
+
+export const db = new firestore.Firestore({
+  projectId: config.googleProjectId,
+  credentials: config.googleApplicationCredentials
+})
+export const images = db.collection(config.firebaseCollectionImages)
+
+export async function get<T extends types.Model>(
+  doc: firestore.DocumentReference,
+  userId?: string
+): Promise<T> {
+  const snapshot = await doc.get()
+
+  if (snapshot.exists) {
+    const res = getSnapshot<T>(snapshot)
+
+    if (userId && res.userId && res.userId !== userId) {
+      throw {
+        message: 'Unauthorized',
+        status: 403
+      }
+    }
+
+    return res
+  }
+
+  throw {
+    message: 'Not found',
+    status: 404
+  }
+}
+
+export function getSnapshot<T extends types.Model>(
+  snapshot: firestore.DocumentSnapshot<firestore.DocumentData>
+): T {
+  const data = snapshot.data()
+  delete data.timestamp
+
+  return {
+    ...data,
+    id: snapshot.id,
+    createdAt: (snapshot.createTime.toDate().getTime() / 1000) | 0,
+    updatedAt: (snapshot.updateTime.toDate().getTime() / 1000) | 0
+  } as T
+}

lib/env.ts

@@ -0,0 +1,52 @@
+/**
+ * All app config that needs to be available server-side.
+ *
+ * @see config.ts for client-side version.
+ */
+
+import { getEnv } from './get-env'
+import { isDev } from './config'
+
+export * from './config'
+
+export const port = getEnv('PORT', '3000')
+export const domain = getEnv('DOMAIN')
+export const host = isDev ? `http://localhost:${port}` : `https://${domain}`
+
+export const apiBaseUrl = `${host}/api`
+export const api = {
+  createPreviewImage: `${apiBaseUrl}/create-preview-image`
+}
+
+export const googleProjectId = getEnv('GCLOUD_PROJECT')
+
+export let googleApplicationCredentials
+
+// this hack is necessary because vercel doesn't support secret files so we need to encode our google
+// credentials a base64-encoded string of the JSON-ified content
+try {
+  const googleApplicationCredentialsBase64 = getEnv(
+    'GOOGLE_APPLICATION_CREDENTIALS'
+  )
+  googleApplicationCredentials = JSON.parse(
+    Buffer.from(googleApplicationCredentialsBase64, 'base64').toString()
+  )
+} catch (err) {
+  console.error(
+    'Firebase config error: invalid "GOOGLE_APPLICATION_CREDENTIALS" should be base64-encoded JSON\n'
+  )
+  throw err
+}
+
+export const firebaseCollectionImages = getEnv('FIREBASE_COLLECTION_IMAGES')
+
+export const notionRootPageId = getEnv('NOTION_ROOT_PAGE_ID')
+
+export const siteName = getEnv('SITE_NAME', 'Transitive Bullshit')
+export const siteDesc = getEnv(
+  'SITE_DESC',
+  'Personal blog and portfolio of Travis Fischer aka Transitive Bullshit.'
+)
+export const siteImage = getEnv('SITE_IMAGE', '/social.jpg')
+export const siteFavicon = getEnv('SITE_FAVICON', '/favicon.png')
+export const siteAuthor = getEnv('SITE_AUTHOR', 'Travis Fischer')

lib/get-all-pages.ts

@@ -0,0 +1,23 @@
+import pMemoize from 'p-memoize'
+import { getAllPagesInSpace, getCanonicalPageId } from 'notion-utils'
+
+import notion from './notion'
+
+export const getAllPages = pMemoize(getAllPagesImpl, { maxAge: 60000 * 5 })
+
+export async function getAllPagesImpl(
+  rootNotionPageId: string,
+  rootNotionSpaceId: string
+): Promise<string[]> {
+  const pages = await getAllPagesInSpace(
+    rootNotionPageId,
+    rootNotionSpaceId,
+    notion.getPage.bind(notion)
+  )
+
+  const canonicalPageIds = Object.keys(pages)
+    .map((pageId) => getCanonicalPageId(pageId, pages[pageId]))
+    .filter(Boolean)
+
+  return canonicalPageIds
+}

lib/get-env.ts

@@ -0,0 +1,17 @@
+export function getEnv(
+  key: string,
+  defaultValue?: string,
+  env = process.env
+): string {
+  const value = env[key]
+
+  if (value !== undefined) {
+    return value
+  }
+
+  if (defaultValue !== undefined) {
+    return defaultValue
+  }
+
+  throw new Error(`Config error: missing required env var "${key}"`)
+}

lib/get-preview-images.ts

@@ -0,0 +1,50 @@
+import crypto from 'crypto'
+import got from 'got'
+
+import { api } from './env'
+import * as types from './types'
+import * as db from './db'
+
+function sha256(input: Buffer | string) {
+  const buffer = Buffer.isBuffer(input) ? input : Buffer.from(input)
+  return crypto.createHash('sha256').update(buffer).digest('hex')
+}
+
+export async function getPreviewImages(
+  images: string[]
+): Promise<types.PreviewImageMap> {
+  const imageDocRefs = images.map((url) => {
+    const id = sha256(url)
+    return db.images.doc(id)
+  })
+
+  if (!imageDocRefs.length) {
+    return {}
+  }
+
+  const imageDocs = await db.db.getAll(...imageDocRefs)
+  const results = imageDocs.map((model, index) => {
+    if (model.exists) {
+      return model.data() as types.PreviewImage
+    } else {
+      // fire and forget
+      got.post(api.createPreviewImage, {
+        json: {
+          url: images[index],
+          id: model.id
+        }
+      })
+    }
+  })
+
+  return results
+    .filter(Boolean)
+    .filter((image) => !image.error)
+    .reduce(
+      (acc, result) => ({
+        ...acc,
+        [result.url]: result
+      }),
+      {}
+    )
+}

lib/get-site-for-domain.ts

@@ -0,0 +1,14 @@
+import * as config from './env'
+import * as types from './types'
+
+export const getSiteForDomain = async (
+  domain: string
+): Promise<types.Site | null> => {
+  return {
+    domain,
+    name: config.siteName,
+    rootNotionPageId: config.notionRootPageId,
+    description: config.siteDesc,
+    image: config.siteImage
+  } as types.Site
+}

lib/get-site-maps.ts

@@ -0,0 +1,32 @@
+import pMap from 'p-map'
+
+import { getAllPages } from './get-all-pages'
+import { getSites } from './get-sites'
+import * as types from './types'
+
+export async function getSiteMaps(): Promise<types.SiteMap[]> {
+  const sites = await getSites()
+
+  const siteMaps = await pMap(
+    sites,
+    async (site, index) => {
+      try {
+        console.log('getSiteMap', index, site)
+        return {
+          site,
+          pageIds: await getAllPages(
+            site.rootNotionPageId,
+            site.rootNotionSpaceId
+          )
+        }
+      } catch (err) {
+        console.warn('site build error', index, site, err)
+      }
+    },
+    {
+      concurrency: 4
+    }
+  )
+
+  return siteMaps.filter(Boolean)
+}

lib/get-sites.ts

@@ -0,0 +1,7 @@
+import { getSiteForDomain } from './get-site-for-domain'
+import * as config from './env'
+import * as types from './types'
+
+export async function getSites(): Promise<types.Site[]> {
+  return [await getSiteForDomain(config.domain)]
+}

lib/map-image-url.ts

@@ -0,0 +1,44 @@
+import { Block } from 'notion-types'
+
+export const mapNotionImageUrl = (url: string, block: Block) => {
+  if (!url) {
+    return null
+  }
+
+  if (url.startsWith('data:')) {
+    return null
+  }
+
+  const origUrl = url
+
+  if (url.startsWith('/images')) {
+    url = `https://www.notion.so${url}`
+  }
+
+  // more recent versions of notion don't proxy unsplash images
+  if (!url.startsWith('https://images.unsplash.com')) {
+    url = `https://www.notion.so${
+      url.startsWith('/image') ? url : `/image/${encodeURIComponent(url)}`
+    }`
+
+    const notionImageUrlV2 = new URL(url)
+    const table = block.parent_table === 'space' ? 'block' : block.parent_table
+    notionImageUrlV2.searchParams.set('table', table)
+    notionImageUrlV2.searchParams.set('id', block.id)
+    notionImageUrlV2.searchParams.set('cache', 'v2')
+
+    url = notionImageUrlV2.toString()
+  }
+
+  // console.log({ url, origUrl })
+  return mapImageUrl(url)
+}
+
+export const mapImageUrl = (imageUrl: string) => {
+  if (imageUrl.startsWith('data:')) {
+    return null
+  }
+
+  // Our proxy uses Cloudflare's global CDN to cache these image assets
+  return `https://ssfy.io/${encodeURIComponent(imageUrl)}`
+}

lib/map-page-url.ts

@@ -0,0 +1,31 @@
+import * as types from './types'
+import { getCanonicalPageId, uuidToId, parsePageId } from 'notion-utils'
+
+export const mapPageUrl = (
+  site: types.Site,
+  recordMap: types.ExtendedRecordMap,
+  searchParams: URLSearchParams
+) => (pageId: string = '') => {
+  if (uuidToId(pageId) === site.rootNotionPageId) {
+    return createUrl('/', searchParams)
+  } else {
+    return createUrl(`/${getCanonicalPageId(pageId, recordMap)}`, searchParams)
+  }
+}
+
+export const getCanonicalPageUrl = (
+  site: types.Site,
+  recordMap: types.ExtendedRecordMap
+) => (pageId: string = '') => {
+  const pageUuid = parsePageId(pageId, { uuid: true })
+
+  if (uuidToId(pageId) === site.rootNotionPageId) {
+    return `https://${site.domain}`
+  } else {
+    return `https://${site.domain}/${getCanonicalPageId(pageUuid, recordMap)}`
+  }
+}
+
+function createUrl(path: string, searchParams: URLSearchParams) {
+  return [path, searchParams.toString()].filter(Boolean).join('?')
+}

lib/mock-db.ts

@@ -0,0 +1,16 @@
+import * as types from './types'
+
+// mock database for testing purposes
+export const sites: Partial<types.Site>[] = [
+  {
+    name: 'Notion2Site Demo',
+    domain: 'localhost',
+    // rootNotionPageId: 'dc6f890fec6b4766bd9b616324904187',
+    rootNotionPageId: '2988138f78424344b67db048e3792229',
+    rootNotionSpaceId: 'fde5ac74-eea3-4527-8f00-4482710e1af3',
+    // fontFamily: 'Oxygen',
+    description: 'This is a demo website powered by Notion2Site.',
+    image: 'https://storage.googleapis.com/saasify-assets/notion2site-v2.jpg',
+    html: `<script>console.log(\`\n\nHello from custom JS injected into this page.\n\n\`)</script>`
+  }
+]

lib/notion.ts

@@ -0,0 +1,63 @@
+import { NotionAPI } from 'notion-client'
+import { ExtendedRecordMap } from 'notion-types'
+import { getPreviewImages } from './get-preview-images'
+import { mapNotionImageUrl } from './map-image-url'
+
+const notion = new NotionAPI({
+  apiBaseUrl: process.env.NOTION_API_BASE_URL
+})
+
+export default notion
+
+export async function getPage(pageId: string): Promise<ExtendedRecordMap> {
+  const recordMap = await notion.getPage(pageId)
+  const blockIds = Object.keys(recordMap.block)
+
+  const imageUrls: string[] = blockIds
+    .map((blockId) => {
+      const block = recordMap.block[blockId]?.value
+
+      if (block) {
+        if (block.type === 'image') {
+          const source = block.properties?.source?.[0]?.[0]
+
+          if (source) {
+            return {
+              block,
+              url: source
+            }
+          }
+        }
+
+        if ((block.format as any)?.page_cover) {
+          const source = (block.format as any).page_cover
+
+          return {
+            block,
+            url: source
+          }
+        }
+      }
+    })
+    .filter(Boolean)
+    .map(({ block, url }) => mapNotionImageUrl(url, block))
+    .filter(Boolean)
+
+  const urls = Array.from(new Set(imageUrls))
+  const previewImageMap = await getPreviewImages(urls)
+  ;(recordMap as any).preview_images = previewImageMap
+
+  return recordMap
+}
+
+// export const getSearch = pMemoize(getSearchImpl, { maxAge: 20000 })
+
+// async function getSearchImpl(
+//   params: types.SearchParams
+// ): Promise<types.NotionSearchResultsType | types.NotionError> {
+//   const url = `${apiBaseUrl}/v1/search?${new URLSearchParams(
+//     params as any
+//   ).toString()}`
+
+//   return fetch(url).then((res) => res.json())
+// }

lib/oembed.ts

@@ -0,0 +1,66 @@
+import { parsePageId, getPageTitle } from 'notion-utils'
+import { getPage } from './notion'
+import * as config from './env'
+
+export const oembed = async ({
+  url,
+  maxWidth,
+  maxHeight,
+  dark = false
+}: {
+  url: string
+  maxWidth?: number
+  maxHeight?: number
+  dark?: boolean
+}) => {
+  // TODO: handle pages with no pageId via domain
+  const pageId = parsePageId(url)
+
+  let title = config.siteName
+  let authorName = config.siteAuthor
+
+  try {
+    const page = await getPage(pageId)
+    const pageTitle = getPageTitle(page)
+    if (pageTitle) title = pageTitle
+
+    const user = page.notion_user[Object.keys(page.notion_user)[0]]?.value
+    const name = [user.given_name, user.family_name]
+      .filter(Boolean)
+      .join(' ')
+      .trim()
+    if (name) authorName = name
+  } catch (err) {
+    // TODO: handle gracefully
+    throw err
+  }
+
+  const params: any = { lite: 'true' }
+  if (dark) {
+    params.dark = 'true'
+  }
+
+  const query = new URLSearchParams(params).toString()
+  const embedUrl = `${config.host}/${pageId}?${query}`
+  const defaultWidth = 800
+  const defaultHeight = 600
+  const width = maxWidth ? Math.min(maxWidth, defaultWidth) : defaultWidth
+  const height = maxHeight ? Math.min(maxHeight, defaultHeight) : defaultHeight
+
+  return {
+    version: '1.0',
+    type: 'rich',
+    provider_name: config.siteName,
+    provider_url: config.host,
+    title,
+    author_name: authorName,
+    url,
+    // TODO
+    // thumbnail_url: 'https://repl.it/public/images/replit-logo-800x600.png',
+    // thumbnail_width: 800,
+    // thumbnail_height: 600,
+    width,
+    height,
+    html: `<iframe src="${embedUrl}" sandbox="allow-forms allow-pointer-lock allow-popups allow-same-origin allow-scripts" width="${width}" height="${height}" frameborder="0"></iframe>`
+  }
+}

lib/resolve-notion-page.ts

@@ -0,0 +1,41 @@
+import * as acl from './acl'
+import * as types from './types'
+import { parsePageId } from 'notion-utils'
+import { getPage } from './notion'
+import { getSiteForDomain } from './get-site-for-domain'
+
+export async function resolveNotionPage(domain: string, rawPageId?: string) {
+  let site: types.Site
+  let pageId: string
+  let recordMap: types.ExtendedRecordMap
+
+  if (rawPageId && rawPageId !== 'index') {
+    pageId = parsePageId(rawPageId)
+
+    if (!pageId) {
+      return {
+        error: {
+          message: `Invalid notion page ID "${rawPageId}"`,
+          statusCode: 404
+        }
+      }
+    }
+
+    const resources = await Promise.all([
+      getSiteForDomain(domain),
+      getPage(pageId)
+    ])
+
+    site = resources[0]
+    recordMap = resources[1]
+  } else {
+    site = await getSiteForDomain(domain)
+    pageId = site.rootNotionPageId
+
+    console.log(site)
+    recordMap = await getPage(pageId)
+  }
+
+  const props = { site, recordMap, pageId }
+  return { ...props, ...(await acl.pageAcl(props)) }
+}

lib/types.ts

@@ -0,0 +1,76 @@
+import { Block, ExtendedRecordMap } from 'notion-types'
+
+export * from 'notion-types'
+
+export interface PageError {
+  message?: string
+  statusCode: number
+}
+
+export interface PageProps {
+  site?: Site
+  recordMap?: ExtendedRecordMap
+  pageId?: string
+  error?: PageError
+}
+
+export interface Model {
+  id: string
+  userId: string
+
+  createdAt: number
+  updatedAt: number
+}
+
+export interface Site extends Model {
+  name: string
+  domain: string
+
+  rootNotionPageId: string
+  rootNotionSpaceId: string
+
+  // settings
+  html?: string
+  fontFamily?: string
+  darkMode?: boolean
+  previewImages?: boolean
+
+  // opengraph metadata
+  description?: string
+  image?: string
+
+  timestamp: Date
+
+  // disabled for payment reasons
+  isDisabled: boolean
+}
+
+export interface SiteMap {
+  site: Site
+  pageIds: string[]
+}
+
+export interface Breadcrumb {
+  block: Block
+  active: boolean
+  pageId: string
+  title: string
+  icon: string
+}
+
+export interface PreviewImage {
+  url: string
+  originalWidth: number
+  originalHeight: number
+  width: number
+  height: number
+  type: string
+  dataURIBase64: string
+
+  error?: string
+  statusCode?: number
+}
+
+export interface PreviewImageMap {
+  [url: string]: PreviewImage
+}