From 3a4be195d5604d75efed48ea2c7a0d63a6fd7ec7 Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Wed, 28 Aug 2024 14:50:57 +0200 Subject: [PATCH] feat: Implement linkwarden sync Signed-off-by: Marcel Klehr --- .github/workflows/tests.yml | 2 + src/lib/Account.ts | 1 + src/lib/adapters/Linkwarden.ts | 330 +++++++++++++++++++++++++++++++++ src/test/test.js | 50 +++-- test/selenium-runner.js | 13 +- 5 files changed, 380 insertions(+), 16 deletions(-) create mode 100644 src/lib/adapters/Linkwarden.ts diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index d90a91b883..342f5e9ae6 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -100,6 +100,7 @@ jobs: - git-html - google-drive - google-drive-encrypted + - linkwarden test-name: - test - benchmark root @@ -253,6 +254,7 @@ jobs: FLOCCUS_TEST_SEED: ${{ github.sha }} GIST_TOKEN: ${{ secrets.GIST_TOKEN }} GOOGLE_API_REFRESH_TOKEN: ${{ secrets.GOOGLE_API_REFRESH_TOKEN }} + LINKWARDEN_TOKEN: ${{ secrets.LINKWARDEN_TOKEN }} APP_VERSION: ${{ matrix.app-version }} run: | npm run test diff --git a/src/lib/Account.ts b/src/lib/Account.ts index a235aefbef..cb72687745 100644 --- a/src/lib/Account.ts +++ b/src/lib/Account.ts @@ -17,6 +17,7 @@ import * as Sentry from '@sentry/vue' declare const DEBUG: boolean // register Adapters +AdapterFactory.register('linkwarden', async() => (await import('./adapters/Linkwarden')).default) AdapterFactory.register('nextcloud-folders', async() => (await import('./adapters/NextcloudBookmarks')).default) AdapterFactory.register('nextcloud-bookmarks', async() => (await import('./adapters/NextcloudBookmarks')).default) AdapterFactory.register('webdav', async() => (await import('./adapters/WebDav')).default) diff --git a/src/lib/adapters/Linkwarden.ts b/src/lib/adapters/Linkwarden.ts new file mode 100644 index 0000000000..13b0d1af8b --- /dev/null +++ b/src/lib/adapters/Linkwarden.ts @@ -0,0 +1,330 @@ +import Adapter from '../interfaces/Adapter' +import { Bookmark, Folder, ItemLocation, TItemLocation } from '../Tree' +import PQueue from 'p-queue' +import { IResource } from '../interfaces/Resource' +import Logger from '../Logger' +import { + AuthenticationError, + CancelledSyncError, HttpError, + NetworkError, ParseResponseError, + RedirectError, + RequestTimeoutError +} from '../../errors/Error' +import { Capacitor, CapacitorHttp as Http } from '@capacitor/core' + +export interface LinkwardenConfig { + type: 'linkwarden' + url: string + username: string + password: string + serverFolder: string, + includeCredentials?: boolean + allowRedirects?: boolean + allowNetwork?: boolean + label?: string +} + +const TIMEOUT = 300000 + +export default class LinkwardenAdapter implements Adapter, IResource { + private server: LinkwardenConfig + private fetchQueue: PQueue + private abortController: AbortController + private abortSignal: AbortSignal + private canceled: boolean + + constructor(server: LinkwardenConfig) { + this.server = server + this.fetchQueue = new PQueue({ concurrency: 12 }) + this.abortController = new AbortController() + this.abortSignal = this.abortController.signal + } + + static getDefaultValues(): LinkwardenConfig { + return { + type: 'linkwarden', + url: 'https://example.org', + username: 'bob', + password: 's3cret', + serverFolder: 'Floccus', + includeCredentials: false, + allowRedirects: false, + allowNetwork: false, + } + } + + acceptsBookmark(bookmark: Bookmark): boolean { + return true + } + + cancel(): void { + this.canceled = true + this.abortController.abort() + } + + setData(data:LinkwardenConfig):void { + this.server = { ...data } + } + + getData(): LinkwardenConfig { + return { ...LinkwardenAdapter.getDefaultValues(), ...this.server } + } + + getLabel(): string { + const data = this.getData() + return data.label || (data.username.includes('@') ? data.username + ' on ' + new URL(data.url).hostname : data.username + '@' + new URL(data.url).hostname) + } + + onSyncComplete(): Promise { + return Promise.resolve(undefined) + } + + onSyncFail(): Promise { + return Promise.resolve(undefined) + } + + onSyncStart(needLock?: boolean, forceLock?: boolean): Promise { + this.canceled = false + return Promise.resolve(undefined) + } + + async createBookmark(bookmark: Bookmark): Promise { + Logger.log('(linkwarden)CREATE', {bookmark}) + const {response} = await this.sendRequest( + 'POST', '/api/v1/links', + 'application/json', + { + url: bookmark.url, + name: bookmark.title, + collection: { + id: bookmark.parentId, + }, + }) + return response.id + } + + async updateBookmark(bookmark: Bookmark): Promise { + Logger.log('(linkwarden)UPDATE', {bookmark}) + const {response: collection} = await this.sendRequest('GET', `/api/v1/collections/${bookmark.parentId}`) + await this.sendRequest( + 'PUT', `/api/v1/links/${bookmark.id}`, + 'application/json', + { + url: bookmark.url, + name: bookmark.title, + tags: [], + collection: { + id: bookmark.parentId, + name: collection.name, + ownerId: collection.ownerId, + }, + }) + } + + async removeBookmark(bookmark: Bookmark): Promise { + Logger.log('(linkwarden)DELETE', {bookmark}) + await this.sendRequest('DELETE', `/api/v1/links/${bookmark.id}`) + } + + async createFolder(folder: Folder): Promise { + Logger.log('(linkwarden)CREATEFOLDER', {folder}) + const {response} = await this.sendRequest( + 'POST', '/api/v1/collections', + 'application/json', + { + name: folder.title, + parentId: folder.parentId, + }) + return response.id + } + + async updateFolder(folder: Folder): Promise { + Logger.log('(linkwarden)UPDATEFOLDER', {folder}) + const {response: collection} = await this.sendRequest('GET', `/api/v1/collections/${folder.id}`) + await this.sendRequest( + 'PUT', `/api/v1/collections/${folder.id}`, + 'application/json', + { + ...collection, + name: folder.title, + parentId: folder.parentId, + }) + } + + async removeFolder(folder: Folder): Promise { + Logger.log('(linkwarden)DELETEFOLDER', {folder}) + await this.sendRequest('DELETE', `/api/v1/collections/${folder.id}`) + } + + async getBookmarksTree(loadAll?: boolean): Promise> { + const {response: links} = await this.sendRequest('GET', `/api/v1/links`) + const {response: collections} = await this.sendRequest('GET', `/api/v1/collections`) + + let rootCollection = collections.find(collection => collection.name === this.server.serverFolder && collection.parentId === null) + if (!rootCollection) { + ({response: rootCollection} = await this.sendRequest( + 'POST', '/api/v1/collections', + 'application/json', + { + name: this.server.serverFolder, + })) + } + + const buildTree = (collection) => { + return new Folder({ + id: collection.id, + title: collection.name, + parentId: collection.parentId, + location: ItemLocation.SERVER, + children: collections + .filter(col => col.parentId === collection.id) + .map(buildTree).concat( + links + .filter(link => link.collectionId === collection.id) + .map(link => new Bookmark({ + id: link.id, + title: link.name, + parentId: link.collectionId, + url: link.url, + location: ItemLocation.SERVER, + })) + ), + }) + } + + return buildTree(rootCollection) + } + + async isAvailable(): Promise { + return true + } + + async sendRequest(verb:string, relUrl:string, type:string = null, body:any = null, returnRawResponse = false):Promise { + const url = this.server.url + relUrl + let res + let timedOut = false + + if (type && type.includes('application/json')) { + body = JSON.stringify(body) + } else if (type && type.includes('application/x-www-form-urlencoded')) { + const params = new URLSearchParams() + for (const [key, value] of Object.entries(body || {})) { + params.set(key, value as any) + } + body = params.toString() + } + + Logger.log(`QUEUING ${verb} ${url}`) + + if (Capacitor.getPlatform() !== 'web') { + return this.sendRequestNative(verb, url, type, body, returnRawResponse) + } + + try { + res = await this.fetchQueue.add(() => { + Logger.log(`FETCHING ${verb} ${url}`) + return Promise.race([ + fetch(url, { + method: verb, + credentials: this.server.includeCredentials ? 'include' : 'omit', + headers: { + ...(type && type !== 'multipart/form-data' && { 'Content-type': type }), + Authorization: 'Bearer ' + this.server.password, + }, + signal: this.abortSignal, + ...(body && !['get', 'head'].includes(verb.toLowerCase()) && { body }), + }), + new Promise((resolve, reject) => + setTimeout(() => { + timedOut = true + reject(new RequestTimeoutError()) + }, TIMEOUT) + ), + ]) + }) + } catch (e) { + if (timedOut) throw e + if (this.canceled) throw new CancelledSyncError() + console.log(e) + throw new NetworkError() + } + + Logger.log(`Receiving response for ${verb} ${url}`) + + if (res.redirected && !this.server.allowRedirects) { + throw new RedirectError() + } + + if (returnRawResponse) { + return res + } + + if (res.status === 401 || res.status === 403) { + throw new AuthenticationError() + } + if (res.status === 503 || res.status > 400) { + throw new HttpError(res.status, verb) + } + let json + try { + json = await res.json() + } catch (e) { + throw new ParseResponseError(e.message) + } + + return json + } + + private async sendRequestNative(verb: string, url: string, type: string, body: any, returnRawResponse: boolean) { + let res + let timedOut = false + try { + res = await this.fetchQueue.add(() => { + Logger.log(`FETCHING ${verb} ${url}`) + return Promise.race([ + Http.request({ + url, + method: verb, + disableRedirects: !this.server.allowRedirects, + headers: { + ...(type && type !== 'multipart/form-data' && { 'Content-type': type }), + Authorization: 'Bearer ' + this.server.password, + }, + responseType: 'json', + ...(body && !['get', 'head'].includes(verb.toLowerCase()) && { data: body }), + }), + new Promise((resolve, reject) => + setTimeout(() => { + timedOut = true + reject(new RequestTimeoutError()) + }, TIMEOUT) + ), + ]) + }) + } catch (e) { + if (timedOut) throw e + console.log(e) + throw new NetworkError() + } + + Logger.log(`Receiving response for ${verb} ${url}`) + + if (res.status < 400 && res.status >= 300) { + throw new RedirectError() + } + + if (returnRawResponse) { + return res + } + + if (res.status === 401 || res.status === 403) { + throw new AuthenticationError() + } + if (res.status === 503 || res.status > 400) { + throw new HttpError(res.status, verb) + } + const json = res.data + + return json + } +} diff --git a/src/test/test.js b/src/test/test.js index 2bca73d2ff..e836874098 100644 --- a/src/test/test.js +++ b/src/test/test.js @@ -150,6 +150,12 @@ describe('Floccus', function() { password: random.float(), refreshToken: CREDENTIALS.password, }, + { + type: 'linkwarden', + url: SERVER, + serverFolder: 'Floccus-' + Math.random(), + ...CREDENTIALS, + }, ] before(async function() { @@ -359,7 +365,8 @@ describe('Floccus', function() { }) ] }), - false + false, + Boolean(account.server.orderFolder) ) const bookmark2 = await browser.bookmarks.create({ @@ -390,7 +397,8 @@ describe('Floccus', function() { }) ] }), - false + false, + Boolean(account.server.orderFolder) ) }) it('should update the server on local changes', async function() { @@ -895,7 +903,8 @@ describe('Floccus', function() { }) ] }), - false + false, + Boolean(account.server.orderFolder) ) }) it('should deduplicate unnormalized URLs without getting stuck', async function() { @@ -913,7 +922,7 @@ describe('Floccus', function() { url: 'http://nextcloud.com/' } const localMark2 = { - title: 'url', + title: 'url2', url: 'https://nextcloud.com' } const fooFolder = await browser.bookmarks.create({ @@ -957,7 +966,8 @@ describe('Floccus', function() { }) ] }), - false + false, + Boolean(account.server.orderFolder) ) }) it('should not fail when moving both folders and contents', async function() { @@ -1019,7 +1029,8 @@ describe('Floccus', function() { }) ] }), - false + false, + Boolean(account.server.orderFolder) ) }) it('should not fail when both moving folders and deleting their contents', async function() { @@ -1091,7 +1102,8 @@ describe('Floccus', function() { }) ] }), - false + false, + Boolean(account.server.orderFolder) ) }) it('should handle strange characters well', async function() { @@ -1380,7 +1392,8 @@ describe('Floccus', function() { }) ] }), - false + false, + Boolean(account.server.orderFolder) ) const localTree = await account.localTree.getBookmarksTree(true) @@ -1406,7 +1419,8 @@ describe('Floccus', function() { }) ] }), - false + false, + Boolean(account.server.orderFolder) ) }) it('should move items successfully when mixing creation and moving (1)', async function() { @@ -1478,7 +1492,8 @@ describe('Floccus', function() { }) ] }), - false + false, + Boolean(account.server.orderFolder) ) const localTree = await account.localTree.getBookmarksTree(true) @@ -1513,7 +1528,8 @@ describe('Floccus', function() { }) ] }), - false + false, + Boolean(account.server.orderFolder) ) }) it('should move items successfully when mixing creation and moving (2)', async function() { @@ -1602,7 +1618,8 @@ describe('Floccus', function() { }) ] }), - false + false, + Boolean(account.server.orderFolder) ) const localTree = await account.localTree.getBookmarksTree(true) @@ -1646,7 +1663,8 @@ describe('Floccus', function() { }) ] }), - false + false, + Boolean(account.server.orderFolder) ) }) it('should move items without creating a folder loop', async function() { @@ -1716,7 +1734,8 @@ describe('Floccus', function() { }) ] }), - false + false, + Boolean(account.server.orderFolder) ) const localTree = await account.localTree.getBookmarksTree(true) @@ -1724,7 +1743,8 @@ describe('Floccus', function() { expectTreeEqual( localTree, tree, - false + false, + Boolean(account.server.orderFolder) ) }) it('should integrate existing items from both sides', async function() { diff --git a/test/selenium-runner.js b/test/selenium-runner.js index 3b0ea7724e..c9a1aea2d1 100644 --- a/test/selenium-runner.js +++ b/test/selenium-runner.js @@ -81,12 +81,23 @@ installConsoleHandler() throw new Error('Unknown browser') } - testUrl += `dist/html/test.html?grep=${process.env.FLOCCUS_TEST}&server=http://${process.env.TEST_HOST}&app_version=${process.env.APP_VERSION}&browser=${process.env.SELENIUM_BROWSER}` + let server = `http://${process.env.TEST_HOST}` + + if (process.env.FLOCCUS_TEST.includes('linkwarden')) { + server = `https://cloud.linkwarden.app` + } + + testUrl += `dist/html/test.html?grep=${process.env.FLOCCUS_TEST}&server=${server}&app_version=${process.env.APP_VERSION}&browser=${process.env.SELENIUM_BROWSER}` if (process.env.FLOCCUS_TEST.includes('google-drive')) { testUrl += `&password=${process.env.GOOGLE_API_REFRESH_TOKEN}` } + if (process.env.FLOCCUS_TEST.includes('linkwarden')) { + testUrl += `&username=mk` + testUrl += `&password=${process.env.LINKWARDEN_TOKEN}` + } + if (process.env.FLOCCUS_TEST_SEED) { testUrl += `&seed=${process.env.FLOCCUS_TEST_SEED}` }