fix(seo/sitemap): change sitemap links to avoid 404s

spotinst · Nov 5, 2020 · ea81235 · ea81235
1 parent f48e9eb
commit ea81235
Show file tree

Hide file tree

Showing 7 changed files with 63 additions and 45 deletions.
diff --git a/package.json b/package.json
@@ -58,6 +58,7 @@
     "webpack": "^4.36.1",
     "webpack-cli": "^4.2.0",
     "webpack-dev-server": "^3.11.0",
+    "xml-formatter": "^2.3.1",
     "yarn": "^1.22.0"
   },
   "engines": {

diff --git a/src/404.html b/src/404.html
@@ -17,43 +17,42 @@
        * For example:
        *   http://www.foo.tld/one/two?a=b&c=d#qwe
        *   [BECOMES]
-       *   http://www.foo.tld/?p=/one/two&q=a=b~and~c=d#qwe
+       *   http://www.foo.tld/?/one/two&q=a=b~and~c=d#qwe
        *
        * Note: This 404.html file must be at least 512 bytes for it to work
        * with Internet Explorer (it is currently > 512 bytes).
        *
        * Note: If you are setting up a GitHub Pages site and not using a
        * custom domain (i.e. your site's address is username.github.io/repo-name),
-       * then you need to set `segmentCount` to 1 in order to keep `/repo-name`
-       * in the path after the redirect.
+       * then you need to set `pathSegmentsToKeep` to 1 in order to keep `/repo-name`
+       * in the path after the redirect. Otherwise, leave pathSegmentsToKeep as 0.
        *
        * For example:
        *   https://username.github.io/repo-name/one/two?a=b&c=d#qwe
        *   [BECOMES]
-       *   https://username.github.io/repo-name/?p=/one/two&q=a=b~and~c=d#qwe
+       *   https://username.github.io/repo-name/?/one/two&q=a=b~and~c=d#qwe
        */
-      let segmentCount = 0;
-      let loc = window.location;
-      let url =
-        loc.protocol +
-        "//" +
-        loc.hostname +
-        (loc.port ? ":" + loc.port : "") +
-        loc.pathname
-          .split("/")
-          .slice(0, 1 + segmentCount)
-          .join("/") +
-        "/?p=/" +
-        loc.pathname
-          .slice(1)
-          .split("/")
-          .slice(segmentCount)
-          .join("/")
-          .replace(/&/g, "~and~") +
-        (loc.search ? "&q=" + loc.search.slice(1).replace(/&/g, "~and~") : "") +
-        loc.hash;
-
-      loc.replace(url);
+      let pathSegmentsToKeep = 0;
+      let l = window.location;
+      l.replace(
+        l.protocol +
+          "//" +
+          l.hostname +
+          (l.port ? ":" + l.port : "") +
+          l.pathname
+            .split("/")
+            .slice(0, 1 + pathSegmentsToKeep)
+            .join("/") +
+          "/?/" +
+          l.pathname
+            .slice(1)
+            .split("/")
+            .slice(pathSegmentsToKeep)
+            .join("/")
+            .replace(/&/g, "~and~") +
+          (l.search ? "&" + l.search.slice(1).replace(/&/g, "~and~") : "") +
+          l.hash
+      );
     </script>
   </head>
   <body></body>

diff --git a/src/index.html b/src/index.html
@@ -67,26 +67,19 @@
        * When the Single Page App is loaded further down in this file,
        * the correct URL will be waiting in the browser's history for
        * the app to route accordingly.
-       *
        */
-      ((loc) => {
-        if (loc.search) {
-          let q = {};
-          loc.search
+      ((l) => {
+        if (l.search[1] === "/") {
+          let decoded = l.search
             .slice(1)
             .split("&")
-            .forEach((v) => {
-              let a = v.split("=");
-              q[a[0]] = a.slice(1).join("=").replace(/~and~/g, "&");
-            });
-          if (q.p !== undefined) {
-            let url =
-              loc.pathname.slice(0, -1) +
-              (q.p || "") +
-              (q.q ? "?" + q.q : "") +
-              loc.hash;
-            window.history.replaceState(null, null, url);
-          }
+            .map((s) => s.replace(/~and~/g, "&"))
+            .join("?");
+          window.history.replaceState(
+            null,
+            null,
+            l.pathname.slice(0, -1) + decoded + l.hash
+          );
         }
       })(window.location);
     </script>

diff --git a/src/robots.txt b/src/robots.txt
@@ -0,0 +1 @@
+Sitemap: https://docs.spot.io/sitemap.xml
diff --git a/src/scripts/sitemap.js b/src/scripts/sitemap.js
@@ -5,6 +5,7 @@ const path = require("path");
 const glob = require("glob");
 const { SitemapStream, streamToPromise } = require("sitemap");
 const { Readable } = require("stream");
+const xmlformatter = require("xml-formatter");
 
 /**
  * Hostname.
@@ -92,7 +93,8 @@ const generateSitemap = async (hostname, dir) => {
 
   // Map files to URL entries.
   const urls = contentFiles.map((file) => {
-    let url = file.replace(dir, "").replace(".md", "").replace("/README", "");
+    let url = file.replace(dir, "").replace(".md", "").replace("/README", "/");
+    if (url !== apiRef) url = `?${url}`; // https://git.io/JTpz9
     let links = langs.map((lang) => ({ url, lang }));
     return { url, links, ...tags };
   });
@@ -111,6 +113,11 @@ const generateSitemap = async (hostname, dir) => {
   // Generate sitemap.
   const sitemapXML = await generateSitemap(hostname, docsDir);
 
+  // Convert the XML into a human readable format (pretty print).
+  const sitemapXMLFormatted = xmlformatter(sitemapXML, {
+    collapseContent: true,
+  });
+
   // Write the generated sitemap.
-  await fs.writeFile(sitemapFile, sitemapXML);
+  await fs.writeFile(sitemapFile, sitemapXMLFormatted);
 })();
diff --git a/webpack.config.js b/webpack.config.js
@@ -55,6 +55,11 @@ module.exports = {
           to: buildDirectory,
           noErrorOnMissing: false,
         },
+        {
+          from: path.resolve(__dirname, srcDirectory, "robots.txt"),
+          to: buildDirectory,
+          noErrorOnMissing: false,
+        },
         {
           from: path.resolve(__dirname, srcDirectory, "sitemap.xml"),
           to: buildDirectory,

diff --git a/yarn.lock b/yarn.lock
@@ -7286,6 +7286,18 @@ xdg-basedir@^3.0.0:
   resolved "https://registry.yarnpkg.com/xdg-basedir/-/xdg-basedir-3.0.0.tgz#496b2cc109eca8dbacfe2dc72b603c17c5870ad4"
   integrity sha1-SWsswQnsqNus/i3HK2A8F8WHCtQ=
 
+xml-formatter@^2.3.1:
+  version "2.3.1"
+  resolved "https://registry.yarnpkg.com/xml-formatter/-/xml-formatter-2.3.1.tgz#79201637dad4fe4b2e58fbd6bf1524ac830ea505"
+  integrity sha512-jvqmM7aowtzgi05LfvWjXLqajeRNu50/dX6Q26a4jwdpMwuCz6FRYRkvqvPBThe7bordClp35jiR/zxRrtM0NQ==
+  dependencies:
+    xml-parser-xo "^3.1.1"
+
+xml-parser-xo@^3.1.1:
+  version "3.1.1"
+  resolved "https://registry.yarnpkg.com/xml-parser-xo/-/xml-parser-xo-3.1.1.tgz#a87d92e44fa8ad3ba7242517df96e6b0893f1f47"
+  integrity sha512-gq1nDlJxjKQpPPZUhLbJ52pghtlB4Rz6LAQULm3SF6xzOYVnUloBglNhJR9vtZB3vIxMN/R3nZTf3qmun+6GCg==
+
 xtend@^4.0.0, xtend@^4.0.1, xtend@~4.0.1:
   version "4.0.2"
   resolved "https://registry.yarnpkg.com/xtend/-/xtend-4.0.2.tgz#bb72779f5fa465186b1f438f674fa347fdb5db54"