From 50dac7dc50f6c5a4fdeeb1d5d5f4324ea6c820f4 Mon Sep 17 00:00:00 2001 From: Ilya Kreymer Date: Tue, 26 Nov 2024 11:17:07 -0800 Subject: [PATCH] 1.12.2 release -> main (#2181) Merge 1.12.2 release changes into main, includes: - Collection replay full refresh on metadata / archived items (#2176) - Fix for self-registration default org (#2178) - Prepend missing https in start URL (#2177) - Updated billing to support free trial messaging (#2179) --------- Co-authored-by: sua yoo Co-authored-by: Henry Wilkinson Co-authored-by: sua yoo Co-authored-by: SuaYoo --- backend/btrixcloud/orgs.py | 3 +- backend/btrixcloud/storages.py | 2 +- chart/values.yaml | 2 +- .../collections/collection-items-dialog.ts | 6 +- .../crawl-workflows/workflow-editor.ts | 124 ++++++++++-------- .../src/features/org/org-status-banner.ts | 89 +++++++++---- frontend/src/pages/org/collection-detail.ts | 19 ++- .../pages/org/settings/components/billing.ts | 69 ++++++---- frontend/src/replayWebPage.d.ts | 6 +- frontend/src/types/billing.ts | 1 + frontend/xliff/es.xlf | 48 +++++-- 11 files changed, 242 insertions(+), 127 deletions(-) diff --git a/backend/btrixcloud/orgs.py b/backend/btrixcloud/orgs.py index ddf1f8a3af..e33c1d7644 100644 --- a/backend/btrixcloud/orgs.py +++ b/backend/btrixcloud/orgs.py @@ -115,6 +115,7 @@ class OrgOps: invites: InviteOps user_manager: UserManager + register_to_org_id: Optional[str] base_crawl_ops: BaseCrawlOps default_primary: Optional[StorageRef] @@ -295,7 +296,7 @@ async def get_default_register_org(self) -> Organization: """Get default organiation for new user registration, or default org""" if self.register_to_org_id: try: - await self.get_org_by_id(UUID(self.register_to_org_id)) + return await self.get_org_by_id(UUID(self.register_to_org_id)) except HTTPException as exc: raise HTTPException( status_code=500, detail="default_register_org_not_found" diff --git a/backend/btrixcloud/storages.py b/backend/btrixcloud/storages.py index cee943dcbf..50b9557a92 100644 --- a/backend/btrixcloud/storages.py +++ b/backend/btrixcloud/storages.py @@ -143,7 +143,7 @@ def _create_s3_storage(self, storage: dict[str, str]) -> S3Storage: use_access_for_presign = False else: access_endpoint_url = storage.get("access_endpoint_url") or endpoint_url - use_access_for_presign = True + use_access_for_presign = is_bool(storage.get("use_access_for_presign")) return S3Storage( access_key=storage["access_key"], diff --git a/chart/values.yaml b/chart/values.yaml index e3c146635e..513f8c4162 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -75,7 +75,7 @@ allow_dupe_invites: "0" invite_expire_seconds: 604800 # base url for replayweb.page -rwp_base_url: "https://cdn.jsdelivr.net/npm/replaywebpage@2.1.4/" +rwp_base_url: "https://cdn.jsdelivr.net/npm/replaywebpage@2.2.4/" superuser: # set this to enable a superuser admin diff --git a/frontend/src/features/collections/collection-items-dialog.ts b/frontend/src/features/collections/collection-items-dialog.ts index 80b5b820e7..7ed50d9a9a 100644 --- a/frontend/src/features/collections/collection-items-dialog.ts +++ b/frontend/src/features/collections/collection-items-dialog.ts @@ -555,7 +555,7 @@ export class CollectionItemsDialog extends BtrixElement { let selectionMessage = msg("No changes to save"); if (hasChange) { - const messages = []; + const messages: string[] = []; if (addCount) { messages.push( msg( @@ -565,7 +565,9 @@ export class CollectionItemsDialog extends BtrixElement { } if (removeCount) { messages.push( - str`Adding ${this.localize.number(removeCount)} ${pluralOf("items", removeCount)}`, + msg( + str`Removing ${this.localize.number(removeCount)} ${pluralOf("items", removeCount)}`, + ), ); } diff --git a/frontend/src/features/crawl-workflows/workflow-editor.ts b/frontend/src/features/crawl-workflows/workflow-editor.ts index afa70b4509..06f83cc11a 100644 --- a/frontend/src/features/crawl-workflows/workflow-editor.ts +++ b/frontend/src/features/crawl-workflows/workflow-editor.ts @@ -112,7 +112,6 @@ const DEFAULT_BEHAVIORS = [ "autofetch", "siteSpecific", ]; -const MAX_ADDITIONAL_URLS = 100; const getDefaultProgressState = (hasConfigId = false): ProgressState => { let activeTab: StepName = "crawlSetup"; @@ -163,7 +162,8 @@ function getLocalizedWeekDays() { } function validURL(url: string) { - return /((((https?):(?:\/\/)?)(?:[-;:&=+$,\w]+@)?[A-Za-z0-9.-]+|(?:www\.|[-;:&=+$,\w]+@)[A-Za-z0-9.-]+)((?:\/[+~%/.\w\-_]*)?\??(?:[-+=&;%@.\w_]*)#?(?:[.!/\\\w]*))?)/.test( + // adapted from: https://gist.github.com/dperini/729294 + return /^(?:https?:\/\/)?(?:\S+(?::\S*)?@)?(?:(?!(?:10|127)(?:\.\d{1,3}){3})(?!(?:169\.254|192\.168)(?:\.\d{1,3}){2})(?!172\.(?:1[6-9]|2\d|3[0-1])(?:\.\d{1,3}){2})(?:[1-9]\d?|1\d\d|2[01]\d|22[0-3])(?:\.(?:1?\d{1,2}|2[0-4]\d|25[0-5])){2}(?:\.(?:[1-9]\d?|1\d\d|2[0-4]\d|25[0-4]))|(?:(?:[a-z0-9\u00a1-\uffff][a-z0-9\u00a1-\uffff_-]{0,62})?[a-z0-9\u00a1-\uffff]\.)+(?:[a-z\u00a1-\uffff]{2,}\.?))(?::\d{2,5})?(?:[/?#]\S*)?$/i.test( url, ); } @@ -174,7 +174,8 @@ const urlListToArray = flow( trimArray, ); -const URL_LIST_MAX_URLS = 1000; +//todo: make this customizable, perhaps at deploy time +const URL_LIST_MAX_URLS = 100; type CrawlConfigResponse = { run_now_job?: boolean; @@ -814,6 +815,17 @@ export class WorkflowEditor extends BtrixElement { const text = msg("Please enter a valid URL."); inputEl.helpText = text; inputEl.setCustomValidity(text); + } else if ( + inputEl.value && + !inputEl.value.startsWith("https://") && + !inputEl.value.startsWith("http://") + ) { + this.updateFormState( + { + urlList: "https://" + inputEl.value, + }, + true, + ); } }} > @@ -835,19 +847,8 @@ https://archiveweb.page/guide`} required @keyup=${async (e: KeyboardEvent) => { if (e.key === "Enter") { - const inputEl = e.target as SlInput; - await inputEl.updateComplete; - if (!inputEl.value) return; - const { isValid, helpText } = this.validateUrlList( - inputEl.value, - MAX_ADDITIONAL_URLS, - ); - inputEl.helpText = helpText; - if (isValid) { - inputEl.setCustomValidity(""); - } else { - inputEl.setCustomValidity(helpText); - } + await (e.target as SlInput).updateComplete; + this.doValidateTextArea(e.target); } }} @sl-input=${(e: CustomEvent) => { @@ -857,24 +858,16 @@ https://archiveweb.page/guide`} } }} @sl-change=${async (e: CustomEvent) => { - const inputEl = e.target as SlInput; - if (!inputEl.value) return; - const { isValid, helpText } = this.validateUrlList( - inputEl.value, - MAX_ADDITIONAL_URLS, - ); - inputEl.helpText = helpText; - if (isValid) { - inputEl.setCustomValidity(""); - } else { - inputEl.setCustomValidity(helpText); - } + this.doValidateTextArea(e.target); + }} + @sl-blur=${async (e: CustomEvent) => { + this.doValidateTextArea(e.target); }} > `)} ${this.renderHelpTextCol( msg( - str`The crawler will visit and record each URL listed here. You can enter up to ${this.localize.number(MAX_ADDITIONAL_URLS)} URLs.`, + str`The crawler will visit and record each URL listed here. You can enter up to ${this.localize.number(URL_LIST_MAX_URLS)} URLs.`, ), )} `} @@ -997,6 +990,17 @@ https://archiveweb.page/guide`} const text = msg("Please enter a valid URL."); inputEl.helpText = text; inputEl.setCustomValidity(text); + } else if ( + inputEl.value && + !inputEl.value.startsWith("https://") && + !inputEl.value.startsWith("http://") + ) { + this.updateFormState( + { + primarySeedUrl: "https://" + inputEl.value, + }, + true, + ); } }} > @@ -1099,19 +1103,8 @@ https://example.net`} https://archiveweb.page/images/${"logo.svg"}`} @keyup=${async (e: KeyboardEvent) => { if (e.key === "Enter") { - const inputEl = e.target as SlInput; - await inputEl.updateComplete; - if (!inputEl.value) return; - const { isValid, helpText } = this.validateUrlList( - inputEl.value, - MAX_ADDITIONAL_URLS, - ); - inputEl.helpText = helpText; - if (isValid) { - inputEl.setCustomValidity(""); - } else { - inputEl.setCustomValidity(helpText); - } + await (e.target as SlInput).updateComplete; + this.doValidateTextArea(e.target); } }} @sl-input=${(e: CustomEvent) => { @@ -1121,24 +1114,16 @@ https://archiveweb.page/images/${"logo.svg"}`} } }} @sl-change=${async (e: CustomEvent) => { - const inputEl = e.target as SlInput; - if (!inputEl.value) return; - const { isValid, helpText } = this.validateUrlList( - inputEl.value, - MAX_ADDITIONAL_URLS, - ); - inputEl.helpText = helpText; - if (isValid) { - inputEl.setCustomValidity(""); - } else { - inputEl.setCustomValidity(helpText); - } + this.doValidateTextArea(e.target); + }} + @sl-blur=${async (e: CustomEvent) => { + this.doValidateTextArea(e.target); }} > `)} ${this.renderHelpTextCol( msg( - str`The crawler will visit and record each URL listed here. You can enter up to ${this.localize.number(MAX_ADDITIONAL_URLS)} URLs.`, + str`The crawler will visit and record each URL listed here. You can enter up to ${this.localize.number(URL_LIST_MAX_URLS)} URLs.`, ), )} @@ -1147,6 +1132,21 @@ https://archiveweb.page/images/${"logo.svg"}`} `; }; + private doValidateTextArea(target: EventTarget | null) { + const inputEl = target as SlInput; + if (!inputEl.value) return; + const { isValid, helpText } = this.validateUrlList( + inputEl.value, + URL_LIST_MAX_URLS, + ); + inputEl.helpText = helpText; + if (isValid) { + inputEl.setCustomValidity(""); + } else { + inputEl.setCustomValidity(helpText); + } + } + private renderCrawlLimits() { // Max Pages minimum value cannot be lower than seed count const minPages = Math.max( @@ -2076,6 +2076,20 @@ https://archiveweb.page/images/${"logo.svg"}`} str`Please remove or fix the following invalid URL: ${invalidUrl}`, ); } + if (isValid) { + // auto-add https:// prefix if otherwise a valid URL + let updated = false; + for (let i = 0; i < urlList.length; i++) { + const url = urlList[i]; + if (!url.startsWith("http://") && !url.startsWith("https://")) { + urlList[i] = "https://" + url; + updated = true; + } + } + if (updated) { + this.updateFormState({ urlList: urlList.join("\n") }); + } + } } return { isValid, helpText }; } diff --git a/frontend/src/features/org/org-status-banner.ts b/frontend/src/features/org/org-status-banner.ts index 3137462082..7fb9e0f675 100644 --- a/frontend/src/features/org/org-status-banner.ts +++ b/frontend/src/features/org/org-status-banner.ts @@ -4,6 +4,7 @@ import { html, type TemplateResult } from "lit"; import { customElement } from "lit/decorators.js"; import { BtrixElement } from "@/classes/BtrixElement"; +import { SubscriptionStatus } from "@/types/billing"; import { OrgReadOnlyReason } from "@/types/org"; type Alert = { @@ -61,16 +62,32 @@ export class OrgStatusBanner extends BtrixElement { execMinutesQuotaReached, } = this.org; + let daysDiff = 0; + let dateStr = ""; + const futureCancelDate = subscription?.futureCancelDate || null; + + if (futureCancelDate) { + daysDiff = differenceInDays(new Date(), new Date(futureCancelDate)); + + dateStr = this.localize.date(futureCancelDate, { + month: "long", + day: "numeric", + year: "numeric", + hour: "numeric", + }); + } + + const isTrial = subscription?.status === SubscriptionStatus.Trialing; + + // show banner if < this many days of trial is left + const MAX_TRIAL_DAYS_SHOW_BANNER = 4; + return [ { test: () => - !readOnly && !readOnlyOnCancel && !!subscription?.futureCancelDate, + !readOnly && !readOnlyOnCancel && !!futureCancelDate && !isTrial, content: () => { - const daysDiff = differenceInDays( - new Date(), - new Date(subscription!.futureCancelDate!), - ); return { title: daysDiff > 1 @@ -82,15 +99,7 @@ export class OrgStatusBanner extends BtrixElement { detail: html`

${msg( - str`Your subscription ends on ${this.localize.date( - subscription!.futureCancelDate!, - { - month: "long", - day: "numeric", - year: "numeric", - hour: "numeric", - }, - )}. Your user account, org, and all associated data will be deleted.`, + str`Your subscription ends on ${dateStr}. Your user account, org, and all associated data will be deleted.`, )}

@@ -106,13 +115,43 @@ export class OrgStatusBanner extends BtrixElement { }, { test: () => - !readOnly && readOnlyOnCancel && !!subscription?.futureCancelDate, + !readOnly && + !readOnlyOnCancel && + !!futureCancelDate && + isTrial && + daysDiff < MAX_TRIAL_DAYS_SHOW_BANNER, + + content: () => { + return { + title: + daysDiff > 1 + ? msg( + str`You have ${daysDiff} days left of your Browsertrix trial`, + ) + : msg(`Your trial ends within one day`), + + detail: html` +

+ ${msg( + html`Your free trial ends on ${dateStr}. To continue using + Browsertrix, select Choose Plan in + ${billingTabLink}.`, + )} +

+

+ ${msg( + str`Your web archives are always yours — you can download any archived items you'd like to keep + before the trial ends!`, + )} +

+ `, + }; + }, + }, + { + test: () => !readOnly && readOnlyOnCancel && !!futureCancelDate, content: () => { - const daysDiff = differenceInDays( - new Date(), - new Date(subscription!.futureCancelDate!), - ); return { title: daysDiff > 1 @@ -121,20 +160,12 @@ export class OrgStatusBanner extends BtrixElement { detail: html`

${msg( - str`Your subscription ends on ${this.localize.date( - subscription!.futureCancelDate!, - { - month: "long", - day: "numeric", - year: "numeric", - hour: "numeric", - }, - )}. You will no longer be able to run crawls, upload files, create browser profiles, or create collections.`, + str`Your subscription ends on ${dateStr}. You will no longer be able to run crawls, upload files, create browser profiles, or create collections.`, )}

${msg( - html`To keep your plan and continue crawling, see + html`To choose a plan and continue using Browsertrix, see ${billingTabLink}.`, )}

diff --git a/frontend/src/pages/org/collection-detail.ts b/frontend/src/pages/org/collection-detail.ts index 88db9604b9..ca54d9b664 100644 --- a/frontend/src/pages/org/collection-detail.ts +++ b/frontend/src/pages/org/collection-detail.ts @@ -60,6 +60,9 @@ export class CollectionDetail extends BtrixElement { @query(".descriptionExpandBtn") private readonly descriptionExpandBtn?: HTMLElement | null; + @query("replay-web-page") + private readonly replayEmbed?: ReplayWebPage | null; + // Use to cancel requests private getArchivedItemsController: AbortController | null = null; @@ -203,6 +206,7 @@ export class CollectionDetail extends BtrixElement { ?open=${this.openDialogName === "editItems"} @sl-hide=${() => (this.openDialogName = undefined)} @btrix-collection-saved=${() => { + this.refreshReplay(); void this.fetchCollection(); void this.fetchArchivedItems(); }} @@ -215,7 +219,10 @@ export class CollectionDetail extends BtrixElement { .collection=${this.collection!} ?open=${this.openDialogName === "editMetadata"} @sl-hide=${() => (this.openDialogName = undefined)} - @btrix-collection-saved=${() => void this.fetchCollection()} + @btrix-collection-saved=${() => { + this.refreshReplay(); + void this.fetchCollection(); + }} > `, @@ -223,6 +230,16 @@ export class CollectionDetail extends BtrixElement { ${this.renderShareDialog()}`; } + private refreshReplay() { + if (this.replayEmbed) { + try { + this.replayEmbed.fullReload(); + } catch (e) { + console.warn("Full reload not available in RWP"); + } + } + } + private getPublicReplayURL() { return new URL( `/api/orgs/${this.orgId}/collections/${this.collectionId}/public/replay.json`, diff --git a/frontend/src/pages/org/settings/components/billing.ts b/frontend/src/pages/org/settings/components/billing.ts index b880be3216..51f83c93a4 100644 --- a/frontend/src/pages/org/settings/components/billing.ts +++ b/frontend/src/pages/org/settings/components/billing.ts @@ -18,7 +18,7 @@ import { tw } from "@/utils/tailwind"; const linkClassList = tw`transition-color text-primary hover:text-primary-500`; const manageLinkClasslist = clsx( linkClassList, - tw`flex items-center gap-2 p-2 text-sm font-semibold leading-none`, + tw`flex cursor-pointer items-center gap-2 p-2 text-sm font-semibold leading-none`, ); @localized() @@ -41,6 +41,10 @@ export class OrgSettingsBilling extends BtrixElement { let label = msg("Manage Billing"); switch (subscription.status) { + case SubscriptionStatus.Trialing: { + label = msg("Choose Plan"); + break; + } case SubscriptionStatus.PausedPaymentFailed: { label = msg("Update Billing"); break; @@ -112,32 +116,41 @@ export class OrgSettingsBilling extends BtrixElement { ${when( this.org, - (org) => - org.subscription?.futureCancelDate - ? html` -
- - - ${msg( + (org) => { + if (!org.subscription?.futureCancelDate) { + return nothing; + } + + const futureCancelDate = html` + `; + + return html` +
+ + + ${org.subscription.status === + SubscriptionStatus.Trialing + ? msg( + html`Your trial will end on ${futureCancelDate} + - Click Choose Plan to + subscribe`, + ) + : msg( html`Your plan will be canceled on - - `, + ${futureCancelDate}`, )} - -
- ` - : nothing, + +
+ `; + }, () => html` `, )}
@@ -245,6 +258,12 @@ export class OrgSettingsBilling extends BtrixElement { `; break; } + case SubscriptionStatus.Trialing: { + statusLabel = html` + ${msg("Trial")} + `; + break; + } case SubscriptionStatus.PausedPaymentFailed: { statusLabel = html` ${msg("Paused, payment failed")} diff --git a/frontend/src/replayWebPage.d.ts b/frontend/src/replayWebPage.d.ts index 34976bea2f..6753a750ed 100644 --- a/frontend/src/replayWebPage.d.ts +++ b/frontend/src/replayWebPage.d.ts @@ -1,4 +1,6 @@ /** + * @TODO Import from replaywebpage once https://github.com/webrecorder/replayweb.page/issues/376 is addressed + * * @attr {String} source * @attr {String} coll * @attr {String} config @@ -7,7 +9,9 @@ * @attr {String} noCache * @attr {String} url */ -class ReplayWebPage {} +class ReplayWebPage { + fullReload(): void {} +} declare global { interface HTMLElementTagNameMap { diff --git a/frontend/src/types/billing.ts b/frontend/src/types/billing.ts index fdaae35030..24d1dfa987 100644 --- a/frontend/src/types/billing.ts +++ b/frontend/src/types/billing.ts @@ -4,6 +4,7 @@ import { apiDateSchema } from "./api"; export enum SubscriptionStatus { Active = "active", + Trialing = "trialing", PausedPaymentFailed = "paused_payment_failed", Cancelled = "cancelled", } diff --git a/frontend/xliff/es.xlf b/frontend/xliff/es.xlf index 14bf1796a6..d20663bd5b 100644 --- a/frontend/xliff/es.xlf +++ b/frontend/xliff/es.xlf @@ -2947,7 +2947,7 @@ days - Your subscription ends on . Your user account, org, and all associated data will be deleted. + Your subscription ends on . Your user account, org, and all associated data will be deleted. We suggest downloading your archived items before they @@ -2961,11 +2961,7 @@ Archiving will be disabled within one day - Your subscription ends on . You will no longer be able to run crawls, upload files, create browser profiles, or create collections. - - - To keep your plan and continue crawling, see - . + Your subscription ends on . You will no longer be able to run crawls, upload files, create browser profiles, or create collections. Archiving is disabled for this org @@ -3671,7 +3667,7 @@ The URL of the page to crawl. - The crawler will visit and record each URL listed here. You can enter up to URLs. + The crawler will visit and record each URL listed here. You can enter up to URLs. If checked, the crawler will visit pages one link away. @@ -3787,10 +3783,6 @@ Choose your preferred language for displaying Browsertrix in your browser. - - Your plan will be canceled on - - Deleting an org will delete all @@ -3808,6 +3800,40 @@ Profiles: + + Trial + + + Removing + + + You have days left of your Browsertrix trial + + + Your trial ends within one day + + + Your free trial ends on . To continue using + Browsertrix, select Choose Plan in + . + + + Your web archives are always yours — you can download any archived items you'd like to keep + before the trial ends! + + + To choose a plan and continue using Browsertrix, see + . + + + Your trial will end on + - Click Choose Plan to + subscribe + + + Your plan will be canceled on + +