1.12.2 release -> main (#2181)

Merge 1.12.2 release changes into main, includes: - Collection replay full refresh on metadata / archived items (#2176) - Fix for self-registration default org (#2178) - Prepend missing https in start URL (#2177) - Updated billing to support free trial messaging (#2179) --------- Co-authored-by: sua yoo <sua@webrecorder.org> Co-authored-by: Henry Wilkinson <henry@wilkinson.graphics> Co-authored-by: sua yoo <sua@suayoo.com> Co-authored-by: SuaYoo <SuaYoo@users.noreply.github.com>
webrecorder · Nov 26, 2024 · 50dac7d · 50dac7d
1 parent 37c0b06
commit 50dac7d
Show file tree

Hide file tree

Showing 11 changed files with 242 additions and 127 deletions.
diff --git a/backend/btrixcloud/orgs.py b/backend/btrixcloud/orgs.py
@@ -115,6 +115,7 @@ class OrgOps:
 
     invites: InviteOps
     user_manager: UserManager
+    register_to_org_id: Optional[str]
     base_crawl_ops: BaseCrawlOps
     default_primary: Optional[StorageRef]
 
@@ -295,7 +296,7 @@ async def get_default_register_org(self) -> Organization:
         """Get default organiation for new user registration, or default org"""
         if self.register_to_org_id:
             try:
-                await self.get_org_by_id(UUID(self.register_to_org_id))
+                return await self.get_org_by_id(UUID(self.register_to_org_id))
             except HTTPException as exc:
                 raise HTTPException(
                     status_code=500, detail="default_register_org_not_found"

diff --git a/backend/btrixcloud/storages.py b/backend/btrixcloud/storages.py
@@ -143,7 +143,7 @@ def _create_s3_storage(self, storage: dict[str, str]) -> S3Storage:
             use_access_for_presign = False
         else:
             access_endpoint_url = storage.get("access_endpoint_url") or endpoint_url
-            use_access_for_presign = True
+            use_access_for_presign = is_bool(storage.get("use_access_for_presign"))
 
         return S3Storage(
             access_key=storage["access_key"],

diff --git a/chart/values.yaml b/chart/values.yaml
@@ -75,7 +75,7 @@ allow_dupe_invites: "0"
 invite_expire_seconds: 604800
 
 # base url for replayweb.page
-rwp_base_url: "https://cdn.jsdelivr.net/npm/replaywebpage@2.1.4/"
+rwp_base_url: "https://cdn.jsdelivr.net/npm/replaywebpage@2.2.4/"
 
 superuser:
   # set this to enable a superuser admin

diff --git a/frontend/src/features/collections/collection-items-dialog.ts b/frontend/src/features/collections/collection-items-dialog.ts
@@ -555,7 +555,7 @@ export class CollectionItemsDialog extends BtrixElement {
     let selectionMessage = msg("No changes to save");
 
     if (hasChange) {
-      const messages = [];
+      const messages: string[] = [];
       if (addCount) {
         messages.push(
           msg(
@@ -565,7 +565,9 @@ export class CollectionItemsDialog extends BtrixElement {
       }
       if (removeCount) {
         messages.push(
-          str`Adding ${this.localize.number(removeCount)} ${pluralOf("items", removeCount)}`,
+          msg(
+            str`Removing ${this.localize.number(removeCount)} ${pluralOf("items", removeCount)}`,
+          ),
         );
       }
 

diff --git a/frontend/src/features/crawl-workflows/workflow-editor.ts b/frontend/src/features/crawl-workflows/workflow-editor.ts
@@ -112,7 +112,6 @@ const DEFAULT_BEHAVIORS = [
   "autofetch",
   "siteSpecific",
 ];
-const MAX_ADDITIONAL_URLS = 100;
 
 const getDefaultProgressState = (hasConfigId = false): ProgressState => {
   let activeTab: StepName = "crawlSetup";
@@ -163,7 +162,8 @@ function getLocalizedWeekDays() {
 }
 
 function validURL(url: string) {
-  return /((((https?):(?:\/\/)?)(?:[-;:&=+$,\w]+@)?[A-Za-z0-9.-]+|(?:www\.|[-;:&=+$,\w]+@)[A-Za-z0-9.-]+)((?:\/[+~%/.\w\-_]*)?\??(?:[-+=&;%@.\w_]*)#?(?:[.!/\\\w]*))?)/.test(
+  // adapted from: https://gist.github.com/dperini/729294
+  return /^(?:https?:\/\/)?(?:\S+(?::\S*)?@)?(?:(?!(?:10|127)(?:\.\d{1,3}){3})(?!(?:169\.254|192\.168)(?:\.\d{1,3}){2})(?!172\.(?:1[6-9]|2\d|3[0-1])(?:\.\d{1,3}){2})(?:[1-9]\d?|1\d\d|2[01]\d|22[0-3])(?:\.(?:1?\d{1,2}|2[0-4]\d|25[0-5])){2}(?:\.(?:[1-9]\d?|1\d\d|2[0-4]\d|25[0-4]))|(?:(?:[a-z0-9\u00a1-\uffff][a-z0-9\u00a1-\uffff_-]{0,62})?[a-z0-9\u00a1-\uffff]\.)+(?:[a-z\u00a1-\uffff]{2,}\.?))(?::\d{2,5})?(?:[/?#]\S*)?$/i.test(
     url,
   );
 }
@@ -174,7 +174,8 @@ const urlListToArray = flow(
   trimArray,
 );
 
-const URL_LIST_MAX_URLS = 1000;
+//todo: make this customizable, perhaps at deploy time
+const URL_LIST_MAX_URLS = 100;
 
 type CrawlConfigResponse = {
   run_now_job?: boolean;
@@ -814,6 +815,17 @@ export class WorkflowEditor extends BtrixElement {
                     const text = msg("Please enter a valid URL.");
                     inputEl.helpText = text;
                     inputEl.setCustomValidity(text);
+                  } else if (
+                    inputEl.value &&
+                    !inputEl.value.startsWith("https://") &&
+                    !inputEl.value.startsWith("http://")
+                  ) {
+                    this.updateFormState(
+                      {
+                        urlList: "https://" + inputEl.value,
+                      },
+                      true,
+                    );
                   }
                 }}
               >
@@ -835,19 +847,8 @@ https://archiveweb.page/guide`}
                 required
                 @keyup=${async (e: KeyboardEvent) => {
                   if (e.key === "Enter") {
-                    const inputEl = e.target as SlInput;
-                    await inputEl.updateComplete;
-                    if (!inputEl.value) return;
-                    const { isValid, helpText } = this.validateUrlList(
-                      inputEl.value,
-                      MAX_ADDITIONAL_URLS,
-                    );
-                    inputEl.helpText = helpText;
-                    if (isValid) {
-                      inputEl.setCustomValidity("");
-                    } else {
-                      inputEl.setCustomValidity(helpText);
-                    }
+                    await (e.target as SlInput).updateComplete;
+                    this.doValidateTextArea(e.target);
                   }
                 }}
                 @sl-input=${(e: CustomEvent) => {
@@ -857,24 +858,16 @@ https://archiveweb.page/guide`}
                   }
                 }}
                 @sl-change=${async (e: CustomEvent) => {
-                  const inputEl = e.target as SlInput;
-                  if (!inputEl.value) return;
-                  const { isValid, helpText } = this.validateUrlList(
-                    inputEl.value,
-                    MAX_ADDITIONAL_URLS,
-                  );
-                  inputEl.helpText = helpText;
-                  if (isValid) {
-                    inputEl.setCustomValidity("");
-                  } else {
-                    inputEl.setCustomValidity(helpText);
-                  }
+                  this.doValidateTextArea(e.target);
+                }}
+                @sl-blur=${async (e: CustomEvent) => {
+                  this.doValidateTextArea(e.target);
                 }}
               ></sl-textarea>
             `)}
             ${this.renderHelpTextCol(
               msg(
-                str`The crawler will visit and record each URL listed here. You can enter up to ${this.localize.number(MAX_ADDITIONAL_URLS)} URLs.`,
+                str`The crawler will visit and record each URL listed here. You can enter up to ${this.localize.number(URL_LIST_MAX_URLS)} URLs.`,
               ),
             )}
           `}
@@ -997,6 +990,17 @@ https://archiveweb.page/guide`}
               const text = msg("Please enter a valid URL.");
               inputEl.helpText = text;
               inputEl.setCustomValidity(text);
+            } else if (
+              inputEl.value &&
+              !inputEl.value.startsWith("https://") &&
+              !inputEl.value.startsWith("http://")
+            ) {
+              this.updateFormState(
+                {
+                  primarySeedUrl: "https://" + inputEl.value,
+                },
+                true,
+              );
             }
           }}
         >
@@ -1099,19 +1103,8 @@ https://example.net`}
 https://archiveweb.page/images/${"logo.svg"}`}
                 @keyup=${async (e: KeyboardEvent) => {
                   if (e.key === "Enter") {
-                    const inputEl = e.target as SlInput;
-                    await inputEl.updateComplete;
-                    if (!inputEl.value) return;
-                    const { isValid, helpText } = this.validateUrlList(
-                      inputEl.value,
-                      MAX_ADDITIONAL_URLS,
-                    );
-                    inputEl.helpText = helpText;
-                    if (isValid) {
-                      inputEl.setCustomValidity("");
-                    } else {
-                      inputEl.setCustomValidity(helpText);
-                    }
+                    await (e.target as SlInput).updateComplete;
+                    this.doValidateTextArea(e.target);
                   }
                 }}
                 @sl-input=${(e: CustomEvent) => {
@@ -1121,24 +1114,16 @@ https://archiveweb.page/images/${"logo.svg"}`}
                   }
                 }}
                 @sl-change=${async (e: CustomEvent) => {
-                  const inputEl = e.target as SlInput;
-                  if (!inputEl.value) return;
-                  const { isValid, helpText } = this.validateUrlList(
-                    inputEl.value,
-                    MAX_ADDITIONAL_URLS,
-                  );
-                  inputEl.helpText = helpText;
-                  if (isValid) {
-                    inputEl.setCustomValidity("");
-                  } else {
-                    inputEl.setCustomValidity(helpText);
-                  }
+                  this.doValidateTextArea(e.target);
+                }}
+                @sl-blur=${async (e: CustomEvent) => {
+                  this.doValidateTextArea(e.target);
                 }}
               ></sl-textarea>
             `)}
             ${this.renderHelpTextCol(
               msg(
-                str`The crawler will visit and record each URL listed here. You can enter up to ${this.localize.number(MAX_ADDITIONAL_URLS)} URLs.`,
+                str`The crawler will visit and record each URL listed here. You can enter up to ${this.localize.number(URL_LIST_MAX_URLS)} URLs.`,
               ),
             )}
           </div>
@@ -1147,6 +1132,21 @@ https://archiveweb.page/images/${"logo.svg"}`}
     `;
   };
 
+  private doValidateTextArea(target: EventTarget | null) {
+    const inputEl = target as SlInput;
+    if (!inputEl.value) return;
+    const { isValid, helpText } = this.validateUrlList(
+      inputEl.value,
+      URL_LIST_MAX_URLS,
+    );
+    inputEl.helpText = helpText;
+    if (isValid) {
+      inputEl.setCustomValidity("");
+    } else {
+      inputEl.setCustomValidity(helpText);
+    }
+  }
+
   private renderCrawlLimits() {
     // Max Pages minimum value cannot be lower than seed count
     const minPages = Math.max(
@@ -2076,6 +2076,20 @@ https://archiveweb.page/images/${"logo.svg"}`}
           str`Please remove or fix the following invalid URL: ${invalidUrl}`,
         );
       }
+      if (isValid) {
+        // auto-add https:// prefix if otherwise a valid URL
+        let updated = false;
+        for (let i = 0; i < urlList.length; i++) {
+          const url = urlList[i];
+          if (!url.startsWith("http://") && !url.startsWith("https://")) {
+            urlList[i] = "https://" + url;
+            updated = true;
+          }
+        }
+        if (updated) {
+          this.updateFormState({ urlList: urlList.join("\n") });
+        }
+      }
     }
     return { isValid, helpText };
   }