feat: update robots.txt to also be auto-generated

pages/robots.txt.tsx

@@ -0,0 +1,14 @@
+import React from 'react'
+
+import { host } from 'lib/config'
+
+export default class Robots extends React.Component {
+  static async getInitialProps({ res }) {
+    res.setHeader('Content-Type', 'text/plain')
+    res.write(`User-agent: *
+Allow: /
+Sitemap: ${host}/sitemap.xml
+`)
+    res.end()
+  }
+}

public/robots.txt

@@ -1,2 +0,0 @@
-User-agent: *
-Allow: /