From ff38efcc84ea583add5f15b8432d3e871a5f79a4 Mon Sep 17 00:00:00 2001 From: Travis Fischer Date: Sun, 27 Mar 2022 06:50:26 -0400 Subject: [PATCH] feat: optimize robots.txt and sitemap.xml --- pages/api/robots.txt.ts | 23 ------------------ pages/api/sitemap.xml.ts | 50 --------------------------------------- pages/robots.txt.tsx | 32 ++++++++++++++++++------- pages/sitemap.xml.tsx | 51 +++++++++++++++++++--------------------- 4 files changed, 47 insertions(+), 109 deletions(-) delete mode 100644 pages/api/robots.txt.ts delete mode 100644 pages/api/sitemap.xml.ts diff --git a/pages/api/robots.txt.ts b/pages/api/robots.txt.ts deleted file mode 100644 index ade7916..0000000 --- a/pages/api/robots.txt.ts +++ /dev/null @@ -1,23 +0,0 @@ -import { NextApiRequest, NextApiResponse } from 'next' - -import { host } from '../../lib/config' - -export default async ( - req: NextApiRequest, - res: NextApiResponse -): Promise => { - if (req.method !== 'GET') { - return res.status(405).send({ error: 'method not allowed' }) - } - - // cache robots.txt for up to 60 seconds - res.setHeader( - 'Cache-Control', - 'public, s-maxage=60, max-age=60, stale-while-revalidate=60' - ) - res.setHeader('Content-Type', 'text/plain') - res.write(`User-agent: * -Sitemap: ${host}/api/sitemap.xml -`) - res.end() -} diff --git a/pages/api/sitemap.xml.ts b/pages/api/sitemap.xml.ts deleted file mode 100644 index 93f63f2..0000000 --- a/pages/api/sitemap.xml.ts +++ /dev/null @@ -1,50 +0,0 @@ -import { NextApiRequest, NextApiResponse } from 'next' - -import { SiteMap } from '../../lib/types' -import { host } from '../../lib/config' -import { getSiteMaps } from '../../lib/get-site-maps' - -export default async ( - req: NextApiRequest, - res: NextApiResponse -): Promise => { - if (req.method !== 'GET') { - return res.status(405).send({ error: 'method not allowed' }) - } - - const siteMaps = await getSiteMaps() - - // cache sitemap for up to one hour - res.setHeader( - 'Cache-Control', - 'public, s-maxage=3600, max-age=3600, stale-while-revalidate=3600' - ) - res.setHeader('Content-Type', 'text/xml') - res.write(createSitemap(siteMaps[0])) - res.end() -} - -const createSitemap = (siteMap: SiteMap) => - ` - - - - ${host} - - - - ${host}/ - - - ${Object.keys(siteMap.canonicalPageMap) - .map((canonicalPagePath) => - ` - - ${host}/${canonicalPagePath} - - `.trim() - ) - .join('')} - - - ` diff --git a/pages/robots.txt.tsx b/pages/robots.txt.tsx index e79c843..270686e 100644 --- a/pages/robots.txt.tsx +++ b/pages/robots.txt.tsx @@ -4,9 +4,10 @@ import { host } from 'lib/config' export const getServerSideProps: GetServerSideProps = async ({ req, res }) => { if (req.method !== 'GET') { res.statusCode = 405 - res.setHeader("Content-Type", "application/json") - res.write(JSON.stringify({ error: "method not allowed" })) + res.setHeader('Content-Type', 'application/json') + res.write(JSON.stringify({ error: 'method not allowed' })) res.end() + return { props: {} } @@ -14,18 +15,31 @@ export const getServerSideProps: GetServerSideProps = async ({ req, res }) => { res.setHeader( 'Cache-Control', - 'public, s-maxage=60, max-age=60, stale-while-revalidate=60' + 'public, s-maxage=3600, max-age=3600, stale-while-revalidate=3600' ) res.setHeader('Content-Type', 'text/plain') - res.write(`User-agent: * - Sitemap: ${host}/sitemap.xml - `) + + // only allow the site to be crawlable on the production deployment + if (process.env.VERCEL_ENV === 'production') { + res.write(`User-agent: * +Allow: / +Disallow: /api/* + +Sitemap: ${host}/sitemap.xml +`) + } else { + res.write(`User-agent: * +Disallow: / + +Sitemap: ${host}/sitemap.xml +`) + } + res.end() + return { props: {} } } -const RobotsTxt: React.FC = () => null - -export default RobotsTxt +export default () => null diff --git a/pages/sitemap.xml.tsx b/pages/sitemap.xml.tsx index 6c30ae6..0f6dce8 100644 --- a/pages/sitemap.xml.tsx +++ b/pages/sitemap.xml.tsx @@ -6,8 +6,8 @@ import { getSiteMaps } from 'lib/get-site-maps' export const getServerSideProps: GetServerSideProps = async ({ req, res }) => { if (req.method !== 'GET') { res.statusCode = 405 - res.setHeader("Content-Type", "application/json") - res.write(JSON.stringify({ error: "method not allowed" })) + res.setHeader('Content-Type', 'application/json') + res.write(JSON.stringify({ error: 'method not allowed' })) res.end() return { props: {} @@ -16,7 +16,7 @@ export const getServerSideProps: GetServerSideProps = async ({ req, res }) => { const siteMaps = await getSiteMaps() - // cache sitemap for up to one hour + // cache for up to one hour res.setHeader( 'Cache-Control', 'public, s-maxage=3600, max-age=3600, stale-while-revalidate=3600' @@ -30,30 +30,27 @@ export const getServerSideProps: GetServerSideProps = async ({ req, res }) => { } } -const createSitemap = ( - siteMap: SiteMap -) => ` - - - ${host} - +const createSitemap = (siteMap: SiteMap) => + ` + + + ${host} + - - ${host}/ - + + ${host}/ + - ${Object.keys(siteMap.canonicalPageMap) - .map((canonicalPagePath) => - ` - - ${host}/${canonicalPagePath} - - `.trim() - ) - .join('')} - - ` + ${Object.keys(siteMap.canonicalPageMap) + .map((canonicalPagePath) => + ` + + ${host}/${canonicalPagePath} + + `.trim() + ) + .join('')} + +` -const SiteMapXml: React.FC = () => null - -export default SiteMapXml +export default () => null