Skip to content

Commit

Permalink
Add more TypeScript documentation comments
Browse files Browse the repository at this point in the history
  • Loading branch information
jjonescz committed Apr 30, 2022
1 parent c851dc5 commit 707b0e7
Show file tree
Hide file tree
Showing 5 changed files with 27 additions and 0 deletions.
1 change: 1 addition & 0 deletions js/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ function* getLogLevelNames() {
for (const name in logger.levels) yield name;
}

/** Entrypoint of the visual extractor CLI app. */
class Program extends Command {
static flags = {
version: flags.version(),
Expand Down
1 change: 1 addition & 0 deletions js/lib/page-pool.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ export interface PagePoolOptions {
disableJavaScript: boolean;
}

/** Creates a pool of reusable Puppeteer pages. */
export function createPagePool(scraper: Scraper, opts: PagePoolOptions) {
const factory = <genericPool.Factory<puppeteer.Page>>{
create: async () => {
Expand Down
1 change: 1 addition & 0 deletions js/lib/scraping-stats.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import { nameOf } from './utils';

/** Simple numerical statistics of scraping. */
export class ScrapingStats {
/** Map from status code to number of occurrences. */
public readonly status: Record<number, number> = {};
Expand Down
14 changes: 14 additions & 0 deletions js/lib/utils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ export function nameOf<T>(name: Extract<keyof T, string>): string {
return name;
}

/** Replaces file extension of {@link fullPath} with {@link ext}. */
export function replaceExtension(fullPath: string, ext: string) {
return path.format({
...path.parse(fullPath),
Expand All @@ -21,6 +22,10 @@ export function replaceExtension(fullPath: string, ext: string) {
});
}

/**
* Replaces {@link prefix} inside file name of {@link fullPath} with
* {@link replacement}.
*/
export function replacePrefix(
fullPath: string,
prefix: string,
Expand All @@ -35,6 +40,7 @@ export function replacePrefix(
});
}

/** Adds {@link suffix} to file name (before extension) of {@link fullPath}. */
export function addSuffix(fullPath: string, suffix: string) {
const parsed = path.parse(fullPath);
return path.format({
Expand All @@ -44,6 +50,7 @@ export function addSuffix(fullPath: string, suffix: string) {
});
}

/** Reads file at {@link fullPath} if it exists. */
export async function tryReadFile(fullPath: string, defaultContents: string) {
if (!existsSync(fullPath)) return defaultContents;
return await readFile(fullPath, { encoding: 'utf-8' });
Expand All @@ -61,6 +68,7 @@ export function writeFileSafe(file: string, data: string) {
renameSync(tempFile, file);
}

/** Escapes {@link filePath} to be safe for use in a glob pattern. */
export function escapeFilePath(filePath: string) {
return filePath.replace(/[`$^*+?()[\]]/g, '\\$&');
}
Expand Down Expand Up @@ -95,6 +103,7 @@ export function getHttps(url: string, query: Record<string, string>) {
});
}

/** Normalizes {@link url} so it can be compared for equality. */
export function normalizeUrl(url: string) {
// This removes superfluous port numbers.
try {
Expand All @@ -106,14 +115,17 @@ export function normalizeUrl(url: string) {
}
}

/** Compares two URLs for equality. */
export function urlsEqual(a: string, b: string) {
return normalizeUrl(a) === normalizeUrl(b);
}

/** Iterates over elements of {@link array} along their indices. */
export function enumerate<T>(array: T[]) {
return array.map((v, i) => [i, v] as const);
}

/** Removes elements from {@link array} where {@link predicate} holds. */
export function removeWhere<T>(
array: T[],
predicate: (item: T, index: number) => boolean
Expand All @@ -126,12 +138,14 @@ export function removeWhere<T>(
}
}

/** Parses {@link input} into an integer if possible. */
export function tryParseInt(input: any, defaultValue: number): number {
const result = parseInt(input?.toString());
if (isNaN(result)) return defaultValue;
return result;
}

/** Constructs temporary file path with the given file {@link extension}. */
export async function temporaryFilePath(extension: string) {
await mkdir(TEMPORARY_DIR, { recursive: true });
const num = crypto.randomBytes(8).readBigUInt64LE(0);
Expand Down
10 changes: 10 additions & 0 deletions js/lib/wayback.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,15 +19,21 @@ export class Wayback {
*/
private responses: Record<string, string | null> = {};

/** Variant to use for {@link getArchiveUrl}. */
public variant = 'id_';

/**
* Constructs URL to Wayback Machine displaying {@link url} at
* {@link timestamp}.
*/
public getArchiveUrl(url: string, timestamp: string) {
url = normalizeUrl(url);
// For URL scheme, see
// https://en.wikipedia.org/wiki/Help:Using_the_Wayback_Machine#Specific_archive_copy.
return `https://web.archive.org/web/${timestamp}${this.variant}/${url}`;
}

/** Parses Wayback Machine {@link url} scheme. */
public parseArchiveUrl(url: string) {
url = normalizeUrl(url);
const match = url.match(ARCHIVE_URL_REGEX);
Expand All @@ -36,6 +42,10 @@ export class Wayback {
return [date, pageUrl] as const;
}

/**
* Determines whether {@link request} corresponds to redirect of Wayback
* Machine.
*/
public isArchiveRedirect(request: HTTPRequest) {
const archive = this.parseArchiveUrl(request.url());
if (archive == null) return null;
Expand Down

0 comments on commit 707b0e7

Please sign in to comment.