-
-
Notifications
You must be signed in to change notification settings - Fork 1.5k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* 404 linter script * 404 linter github workflow * temporarily run linter it's own PR branch * fix file path * skip run if both docs (user facing and dev docs) skip build * fix wording * run a a prod server on localhost before 404 linting * fix commands * fix job name * change * dedupe job name * add progress bar * [getsentry/action-github-commit] Auto commit * use cli-progress for the prgoress bar * add --progress flag to avoid stdout spam * fix: update remaining 404s (#10853) * docs: add section for custom otel sampler (#10843) * fix: remaining 404s * fix 404 * chagne report working * remove unused dependency * remove phony change * switch target branch to master --------- Co-authored-by: getsantry[bot] <66042841+getsantry[bot]@users.noreply.github.com> Co-authored-by: Charly Gomez <charly.gomez@sentry.io>
- Loading branch information
3 people
authored
Jul 25, 2024
1 parent
3941035
commit 7f971f7
Showing
10 changed files
with
253 additions
and
10 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,59 @@ | ||
| name: Lint Docs for 404s | ||
|
|
||
| on: | ||
| push: | ||
| branches: [master] | ||
| pull_request: | ||
| branches: [master] | ||
|
|
||
| jobs: | ||
| index: | ||
| runs-on: ubuntu-latest | ||
| steps: | ||
| - uses: actions/checkout@v2 | ||
| - uses: getsentry/action-setup-volta@c52be2ea13cfdc084edb806e81958c13e445941e # v1.2.0 | ||
| - uses: dorny/paths-filter@v3 | ||
| id: filter | ||
| with: | ||
| filters: | | ||
| docs: | ||
| - 'docs/**' | ||
| - 'includes/**' | ||
| - 'platform-includes/**' | ||
| dev-docs: | ||
| - 'develop-docs/**' | ||
| - uses: oven-sh/setup-bun@v1 | ||
| with: | ||
| bun-version: latest | ||
|
|
||
| - uses: actions/cache@v4 | ||
| id: cache | ||
| with: | ||
| path: ${{ github.workspace }}/node_modules | ||
| key: node-${{ runner.os }}-${{ hashFiles('**/yarn.lock') }} | ||
|
|
||
| - run: yarn install --frozen-lockfile | ||
| if: steps.cache.outputs.cache-hit != 'true' | ||
|
|
||
| # Remove the changelog directory to avoid a build error due to missing `DATABASE_URL` | ||
| # and save some build time. | ||
| - run: rm -r app/changelog | ||
|
|
||
| - run: yarn build | ||
| if: steps.filter.outputs.docs == 'true' | ||
|
|
||
| - run: yarn build:developer-docs | ||
| if: steps.filter.outputs.dev-docs == 'true' | ||
|
|
||
| - name: Start Http Server | ||
| run: yarn start & | ||
| if: steps.filter.outputs.docs == 'true' || steps.filter.outputs.dev-docs == 'true' | ||
|
|
||
| - name: Lint 404s | ||
| run: bun ./scripts/lint-404s/main.ts | ||
| if: steps.filter.outputs.docs == 'true' || steps.filter.outputs.dev-docs == 'true' | ||
|
|
||
| - name: Kill Http Server | ||
| run: kill $(lsof -t -i:3000) || true | ||
| if: steps.filter.outputs.docs == 'true' || steps.filter.outputs.dev-docs == 'true' | ||
| continue-on-error: true |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,2 @@ | ||
| / | ||
| /changelog/ |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,155 @@ | ||
| /* eslint-disable no-console */ | ||
|
|
||
| import {readFileSync} from 'fs'; | ||
| import path, {dirname} from 'path'; | ||
| import {fileURLToPath} from 'url'; | ||
|
|
||
| const baseURL = 'http://localhost:3000/'; | ||
| type Link = {href: string; innerText: string}; | ||
|
|
||
| const trimSlashes = (s: string) => s.replace(/(^\/|\/$)/g, ''); | ||
|
|
||
| // @ts-ignore | ||
| const ignoreListFile = path.join(dirname(import.meta.url), './ignore-list.txt'); | ||
|
|
||
| const showProgress = process.argv.includes('--progress'); | ||
|
|
||
| // Paths to skip | ||
| const ignoreList: string[] = readFileSync(fileURLToPath(ignoreListFile), 'utf8') | ||
| .split('\n') | ||
| .map(trimSlashes) | ||
| .filter(Boolean); | ||
|
|
||
| async function fetchWithFollow(url: URL | string): Promise<Response> { | ||
| const r = await fetch(url); | ||
| if (r.status >= 300 && r.status < 400 && r.headers.has('location')) { | ||
| return fetchWithFollow(r.headers.get('location')!); | ||
| } | ||
| return r; | ||
| } | ||
|
|
||
| async function main() { | ||
| const sitemap = await fetch(`${baseURL}sitemap.xml`).then(r => r.text()); | ||
|
|
||
| const slugs = [...sitemap.matchAll(/<loc>([^<]*)<\/loc>/g)] | ||
| .map(l => l[1]) | ||
| .map(url => trimSlashes(new URL(url).pathname)) | ||
| .filter(Boolean); | ||
| const allSlugsSet = new Set(slugs); | ||
|
|
||
| console.log('Checking 404s on %d pages', slugs.length); | ||
|
|
||
| const all404s: {page404s: Link[]; slug: string}[] = []; | ||
|
|
||
| // check if the slug equivalent of the href is in the sitemap | ||
| const isInSitemap = (href: string) => { | ||
| // remove hash | ||
| const pathnameSlug = trimSlashes(href.replace(/#.*$/, '')); | ||
|
|
||
| // some #hash links result in empty slugs when stripped | ||
| return pathnameSlug === '' || allSlugsSet.has(pathnameSlug); | ||
| }; | ||
|
|
||
| function shoudlSkipLink(href: string) { | ||
| const isExternal = (href_: string) => | ||
| href_.startsWith('http') || href_.startsWith('mailto:'); | ||
| const isLocalhost = (href_: string) => | ||
| href_.startsWith('http') && new URL(href_).hostname === 'localhost'; | ||
| const isIp = (href_: string) => /(\d{1,3}\.){3}\d{1,3}/.test(href_); | ||
| const isImage = (href_: string) => /\.(png|jpg|jpeg|gif|svg|webp)$/.test(href_); | ||
|
|
||
| return [ | ||
| isExternal, | ||
| (s = '') => ignoreList.includes(trimSlashes(s)), | ||
| isImage, | ||
| isLocalhost, | ||
| isIp, | ||
| ].some(fn => fn(href)); | ||
| } | ||
|
|
||
| async function is404(link: Link, pageUrl: URL): Promise<boolean> { | ||
| if (shoudlSkipLink(link.href)) { | ||
| return false; | ||
| } | ||
|
|
||
| const fullPath = link.href.startsWith('/') | ||
| ? trimSlashes(link.href) | ||
| : // relative path | ||
| trimSlashes(new URL(pageUrl.pathname + '/' + link.href, baseURL).pathname); | ||
|
|
||
| if (isInSitemap(fullPath)) { | ||
| return false; | ||
| } | ||
| const fullUrl = new URL(fullPath, baseURL); | ||
| const resp = await fetchWithFollow(fullUrl); | ||
| if (resp.status === 404) { | ||
| return true; | ||
| } | ||
| return false; | ||
| } | ||
|
|
||
| for (const slug of slugs) { | ||
| const pageUrl = new URL(slug, baseURL); | ||
| const now = performance.now(); | ||
| const html = await fetchWithFollow(pageUrl.href).then(r => r.text()); | ||
|
|
||
| const linkRegex = /<a[^>]*href="([^"]*)"[^>]*>([^<]*)<\/a>/g; | ||
| const links = Array.from(html.matchAll(linkRegex)).map(m => { | ||
| const [, href, innerText] = m; | ||
| return {href, innerText}; | ||
| }); | ||
| const page404s = ( | ||
| await Promise.all( | ||
| links.map(async link => { | ||
| const is404_ = await is404(link, pageUrl); | ||
| return [link, is404_] as [Link, boolean]; | ||
| }) | ||
| ) | ||
| ) | ||
| .filter(([_, is404_]) => is404_) | ||
| .map(([link]) => link); | ||
|
|
||
| if (page404s.length) { | ||
| all404s.push({slug, page404s}); | ||
| } | ||
|
|
||
| if (showProgress) { | ||
| console.log( | ||
| page404s.length ? '❌' : '✅', | ||
| `in ${(performance.now() - now).toFixed(1).padStart(4, '0')} ms | ${slug}` | ||
| ); | ||
| } | ||
| } | ||
|
|
||
| if (all404s.length === 0) { | ||
| console.log('\n\n🎉 No 404s found'); | ||
| return false; | ||
| } | ||
| const numberOf404s = all404s.map(x => x.page404s.length).reduce((a, b) => a + b, 0); | ||
| console.log( | ||
| '\n❌ Found %d %s across %d %s', | ||
| numberOf404s, | ||
| numberOf404s === 1 ? '404' : '404s', | ||
| all404s.length, | ||
| all404s.length === 1 ? 'page' : 'pages' | ||
| ); | ||
| for (const {slug, page404s} of all404s) { | ||
| console.log('\n🌐', baseURL + slug); | ||
| for (const link of page404s) { | ||
| console.log(` - [${link.innerText}](${link.href})`); | ||
| } | ||
| } | ||
|
|
||
| console.log( | ||
| '\n👉 Note: the markdown syntax is not necessarily present on the source files, but the links do exist on the final pages' | ||
| ); | ||
| // signal error | ||
| return true; | ||
| } | ||
| const now = performance.now(); | ||
| main().then(has404s => { | ||
| console.log(`\n Done in ${(performance.now() - now).toFixed(1)} ms`); | ||
| process.exit(has404s ? 1 : 0); | ||
| }); | ||
|
|
||
| export {}; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters