diff --git a/README.md b/README.md index 79e9820..416e3d0 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,9 @@ Publish your notes to the web, for free. In your own personal garden. +![image](https://github.com/user-attachments/assets/101e7558-9d0c-452f-8859-f4bcedd26796) + + ## Docs Documentation and examples can be found at [dg-docs.ole.dev](https://dg-docs.ole.dev/). diff --git a/main.ts b/main.ts index 7b9fd97..0a0a106 100644 --- a/main.ts +++ b/main.ts @@ -238,84 +238,39 @@ export default class DigitalGarden extends Plugin { imagesToDelete.length, ); - let errorFiles = 0; - let errorDeleteFiles = 0; - let errorDeleteImage = 0; - new Notice( `Publishing ${filesToPublish.length} notes, deleting ${filesToDelete.length} notes and ${imagesToDelete.length} images. See the status bar in lower right corner for progress.`, 8000, ); - for (const file of filesToPublish) { - try { - statusBar.increment(); - await publisher.publish(file); - } catch { - errorFiles++; + await publisher.publishBatch(filesToPublish); + statusBar.incrementMultiple(filesToPublish.length); - new Notice( - `Unable to publish note ${file.file.name}, skipping it.`, - ); - } + for (const file of filesToDelete) { + await publisher.deleteNote(file.path); + statusBar.increment(); } - for (const filePath of filesToDelete) { - try { - statusBar.increment(); - - // TODO: include sha from file.remoteHash to make faster! - await publisher.deleteNote( - filePath.path, - filePath.sha, - ); - } catch { - errorDeleteFiles++; - - new Notice( - `Unable to delete note ${filePath}, skipping it.`, - ); - } - } - - for (const filePath of imagesToDelete) { - try { - statusBar.increment(); - - await publisher.deleteImage( - filePath.path, - filePath.sha, - ); - } catch { - errorDeleteImage++; - - new Notice( - `Unable to delete image ${filePath}, skipping it.`, - ); - } + for (const image of imagesToDelete) { + await publisher.deleteImage(image.path); + statusBar.increment(); } statusBar.finish(8000); new Notice( - `Successfully published ${ - filesToPublish.length - errorFiles - } notes to your garden.`, + `Successfully published ${filesToPublish.length} notes to your garden.`, ); if (filesToDelete.length > 0) { new Notice( - `Successfully deleted ${ - filesToDelete.length - errorDeleteFiles - } notes from your garden.`, + `Successfully deleted ${filesToDelete.length} notes from your garden.`, ); } if (imagesToDelete.length > 0) { new Notice( - `Successfully deleted ${ - imagesToDelete.length - errorDeleteImage - } images from your garden.`, + `Successfully deleted ${imagesToDelete.length} images from your garden.`, ); } } catch (e) { diff --git a/manifest-beta.json b/manifest-beta.json index 0812ada..e2d70cc 100644 --- a/manifest-beta.json +++ b/manifest-beta.json @@ -1,7 +1,7 @@ { "id": "digitalgarden", "name": "Digital Garden", - "version": "2.56.2", + "version": "2.57.2", "minAppVersion": "0.12.0", "description": "Publish your notes to the web for others to enjoy. For free.", "author": "Ole Eskild Steensen", diff --git a/manifest.json b/manifest.json index 0812ada..e2d70cc 100644 --- a/manifest.json +++ b/manifest.json @@ -1,7 +1,7 @@ { "id": "digitalgarden", "name": "Digital Garden", - "version": "2.56.2", + "version": "2.57.2", "minAppVersion": "0.12.0", "description": "Publish your notes to the web for others to enjoy. For free.", "author": "Ole Eskild Steensen", diff --git a/src/compiler/GardenPageCompiler.ts b/src/compiler/GardenPageCompiler.ts index 1dc54ac..ec08fe4 100644 --- a/src/compiler/GardenPageCompiler.ts +++ b/src/compiler/GardenPageCompiler.ts @@ -30,6 +30,7 @@ import { import Logger from "js-logger"; import { DataviewCompiler } from "./DataviewCompiler"; import { PublishFile } from "../publishFile/PublishFile"; +import { replaceBlockIDs } from "./replaceBlockIDs"; export interface Asset { path: string; @@ -144,21 +145,7 @@ export class GardenPageCompiler { }; createBlockIDs: TCompilerStep = () => (text: string) => { - const block_pattern = / \^([\w\d-]+)/g; - const complex_block_pattern = /\n\^([\w\d-]+)\n/g; - - text = text.replace( - complex_block_pattern, - (_match: string, $1: string) => { - return `{ #${$1}}\n\n`; - }, - ); - - text = text.replace(block_pattern, (match: string, $1: string) => { - return `\n{ #${$1}}\n`; - }); - - return text; + return replaceBlockIDs(text); }; removeObsidianComments: TCompilerStep = () => (text) => { diff --git a/src/compiler/createBlockIDs.test.ts b/src/compiler/createBlockIDs.test.ts new file mode 100644 index 0000000..b05ab68 --- /dev/null +++ b/src/compiler/createBlockIDs.test.ts @@ -0,0 +1,65 @@ +import { replaceBlockIDs } from "./replaceBlockIDs"; + +describe("replaceBlockIDs", () => { + it("should replace block IDs in markdown", () => { + const EXPECTED_MARKDOWN = ` + # header + + foo ^block-id-1234 + + bar ^block-id-5678 + `; + + const result = replaceBlockIDs(EXPECTED_MARKDOWN); + + expect(result).toBe(` + # header + + foo +{ #block-id-1234} + + + bar +{ #block-id-5678} + + `); + }); + + it("should not replace block IDs in code blocks", () => { + const CODEBLOCK_WITH_BLOCKIDS = ` +\`\`\` +foobar. +this is a code block. +but it contains a block ID to try to fool the compiler +and, consequently, wreck your garden. +here it goes: ^block-id-1234 +and for fun, here's another: ^block-id-5678 +\`\`\` + +additionally, block IDs outside of code blocks should be replaced +for example, ^block-id-9999 +and ^block-id-0000 + `; + + const result = replaceBlockIDs(CODEBLOCK_WITH_BLOCKIDS); + + expect(result).toBe(` +\`\`\` +foobar. +this is a code block. +but it contains a block ID to try to fool the compiler +and, consequently, wreck your garden. +here it goes: ^block-id-1234 +and for fun, here's another: ^block-id-5678 +\`\`\` + +additionally, block IDs outside of code blocks should be replaced +for example, +{ #block-id-9999} + +and +{ #block-id-0000} + + `); + }); +}); diff --git a/src/compiler/replaceBlockIDs.ts b/src/compiler/replaceBlockIDs.ts new file mode 100644 index 0000000..50eb1a6 --- /dev/null +++ b/src/compiler/replaceBlockIDs.ts @@ -0,0 +1,35 @@ +export function replaceBlockIDs(markdown: string) { + const block_pattern = / \^([\w\d-]+)/g; + const complex_block_pattern = /\n\^([\w\d-]+)\n/g; + + // To ensure code blocks are not modified... + const codeBlockPattern = /```[\s\S]*?```/g; + + // Extract code blocks and replace them with placeholders + const codeBlocks: string[] = []; + + markdown = markdown.replace(codeBlockPattern, (match) => { + codeBlocks.push(match); + + return `{{CODE_BLOCK_${codeBlocks.length - 1}}}`; + }); + + // Replace patterns outside code blocks + markdown = markdown.replace( + complex_block_pattern, + (_match: string, $1: string) => { + return `{ #${$1}}\n\n`; + }, + ); + + markdown = markdown.replace(block_pattern, (_match: string, $1: string) => { + return `\n{ #${$1}}\n`; + }); + + // Reinsert code blocks + codeBlocks.forEach((block, index) => { + markdown = markdown.replace(`{{CODE_BLOCK_${index}}}`, block); + }); + + return markdown; +} diff --git a/src/dg-testVault/015 Code blocks.md b/src/dg-testVault/015 Code blocks.md new file mode 100644 index 0000000..d6fe2b6 --- /dev/null +++ b/src/dg-testVault/015 Code blocks.md @@ -0,0 +1,26 @@ +--- +dg-publish: true +--- +These codeblocks should not be modified upon publish. + +Sample 1 +```jinja2 +{{ highlight_text }}{% if highlight_location and highlight_location_url %} ([via]({{highlight_location_url}})){% elif highlight_location %} ({{highlight_location}}){% endif %} ^rwhi{{highlight_id}} +{% if highlight_note %} +{{ highlight_note }} ^rwhi{{highlight_id}}-note +{% endif %} +``` + +Sample 2 +```md +In medieval Latin a florilegium (plural florilegia) was a compilation of excerpts from other writings. + The word is from the Latin flos (flower) and legere (to gather): literally a gathering of flowers, or collection of fine extracts from the body of a larger work. ([via](https://en.wikipedia.org/wiki/Florilegium)) ^rwhi724352030 +``` + +Sample 3 +``` +This codeblock has a transclusion syntax in it. +Check it out: ![[001 Links]] +``` + +And for sanity, here's some block references outside of code blocks: foobar ^test-123 \ No newline at end of file diff --git a/src/publisher/Publisher.ts b/src/publisher/Publisher.ts index 6562cd1..12d0387 100644 --- a/src/publisher/Publisher.ts +++ b/src/publisher/Publisher.ts @@ -99,7 +99,7 @@ export default class Publisher { return await this.delete(path, sha); } /** If provided with sha, garden connection does not need to get it seperately! */ - async delete(path: string, sha?: string): Promise { + public async delete(path: string, sha?: string): Promise { this.validateSettings(); const userGardenConnection = new RepositoryConnection({ @@ -115,7 +115,7 @@ export default class Publisher { return !!deleted; } - async publish(file: CompiledPublishFile): Promise { + public async publish(file: CompiledPublishFile): Promise { if (!isPublishFrontmatterValid(file.frontmatter)) { return false; } @@ -133,7 +133,55 @@ export default class Publisher { } } - async uploadToGithub( + public async deleteBatch(filePaths: string[]): Promise { + if (filePaths.length === 0) { + return true; + } + + try { + const userGardenConnection = new RepositoryConnection({ + gardenRepository: this.settings.githubRepo, + githubUserName: this.settings.githubUserName, + githubToken: this.settings.githubToken, + }); + + await userGardenConnection.deleteFiles(filePaths); + + return true; + } catch (error) { + console.error(error); + + return false; + } + } + + public async publishBatch(files: CompiledPublishFile[]): Promise { + const filesToPublish = files.filter((f) => + isPublishFrontmatterValid(f.frontmatter), + ); + + if (filesToPublish.length === 0) { + return true; + } + + try { + const userGardenConnection = new RepositoryConnection({ + gardenRepository: this.settings.githubRepo, + githubUserName: this.settings.githubUserName, + githubToken: this.settings.githubToken, + }); + + await userGardenConnection.updateFiles(filesToPublish); + + return true; + } catch (error) { + console.error(error); + + return false; + } + } + + private async uploadToGithub( path: string, content: string, remoteFileHash?: string, @@ -167,18 +215,18 @@ export default class Publisher { }); } - async uploadText(filePath: string, content: string, sha?: string) { + private async uploadText(filePath: string, content: string, sha?: string) { content = Base64.encode(content); const path = `${NOTE_PATH_BASE}${filePath}`; await this.uploadToGithub(path, content, sha); } - async uploadImage(filePath: string, content: string, sha?: string) { + private async uploadImage(filePath: string, content: string, sha?: string) { const path = `src/site${filePath}`; await this.uploadToGithub(path, content, sha); } - async uploadAssets(assets: Assets) { + private async uploadAssets(assets: Assets) { for (let idx = 0; idx < assets.images.length; idx++) { const image = assets.images[idx]; await this.uploadImage(image.path, image.content, image.remoteHash); diff --git a/src/repositoryConnection/RepositoryConnection.ts b/src/repositoryConnection/RepositoryConnection.ts index 14a9c34..c5cf97f 100644 --- a/src/repositoryConnection/RepositoryConnection.ts +++ b/src/repositoryConnection/RepositoryConnection.ts @@ -1,9 +1,13 @@ import { Octokit } from "@octokit/core"; import Logger from "js-logger"; +import { CompiledPublishFile } from "src/publishFile/PublishFile"; const logger = Logger.get("repository-connection"); const oktokitLogger = Logger.get("octokit"); +const IMAGE_PATH_BASE = "src/site/"; +const NOTE_PATH_BASE = "src/site/notes/"; + interface IOctokitterInput { githubToken: string; githubUserName: string; @@ -157,10 +161,12 @@ export class RepositoryConnection { } } - async getLatestCommit(): Promise<{ sha: string } | undefined> { + async getLatestCommit(): Promise< + { sha: string; commit: { tree: { sha: string } } } | undefined + > { try { const latestCommit = await this.octokit.request( - "GET /repos/{owner}/{repo}/commits/HEAD", + `GET /repos/{owner}/{repo}/commits/HEAD?cacheBust=${Date.now()}`, this.getBasePayload(), ); @@ -194,6 +200,217 @@ export class RepositoryConnection { } } + // NB: Do not use this, it does not work for some reason. + //TODO: Fix this. For now use deleteNote and deleteImage instead + async deleteFiles(filePaths: string[]) { + const latestCommit = await this.getLatestCommit(); + + if (!latestCommit) { + logger.error("Could not get latest commit"); + + return; + } + + const normalizePath = (path: string) => + path.startsWith("/") ? path.slice(1) : path; + + const filesToDelete = filePaths.map((path) => { + if (path.endsWith(".md")) { + return `${NOTE_PATH_BASE}${normalizePath(path)}`; + } + + return `${IMAGE_PATH_BASE}${normalizePath(path)}`; + }); + + const repoDataPromise = this.octokit.request( + "GET /repos/{owner}/{repo}", + { + ...this.getBasePayload(), + }, + ); + + const latestCommitSha = latestCommit.sha; + const baseTreeSha = latestCommit.commit.tree.sha; + + const baseTree = await this.octokit.request( + "GET /repos/{owner}/{repo}/git/trees/{tree_sha}?recursive=1", + { + ...this.getBasePayload(), + tree_sha: baseTreeSha, + }, + ); + + const newTreeEntries = baseTree.data.tree + .filter( + (item: { path: string }) => !filesToDelete.includes(item.path), + ) // Exclude files to delete + .map( + (item: { + path: string; + mode: string; + type: string; + sha: string; + }) => ({ + path: item.path, + mode: item.mode, + type: item.type, + sha: item.sha, + }), + ); + + const newTree = await this.octokit.request( + "POST /repos/{owner}/{repo}/git/trees", + { + ...this.getBasePayload(), + tree: newTreeEntries, + }, + ); + + const commitMessage = "Deleted multiple files"; + + const newCommit = await this.octokit.request( + "POST /repos/{owner}/{repo}/git/commits", + { + ...this.getBasePayload(), + message: commitMessage, + tree: newTree.data.sha, + parents: [latestCommitSha], + }, + ); + + const defaultBranch = (await repoDataPromise).data.default_branch; + + await this.octokit.request( + "PATCH /repos/{owner}/{repo}/git/refs/{ref}", + { + ...this.getBasePayload(), + ref: `heads/${defaultBranch}`, + sha: newCommit.data.sha, + }, + ); + } + + async updateFiles(files: CompiledPublishFile[]) { + const latestCommit = await this.getLatestCommit(); + + if (!latestCommit) { + logger.error("Could not get latest commit"); + + return; + } + + const repoDataPromise = this.octokit.request( + "GET /repos/{owner}/{repo}", + { + ...this.getBasePayload(), + }, + ); + + const latestCommitSha = latestCommit.sha; + const baseTreeSha = latestCommit.commit.tree.sha; + + const normalizePath = (path: string) => + path.startsWith("/") ? path.slice(1) : path; + + const treePromises = files.map(async (file) => { + const [text, _] = file.compiledFile; + + try { + const blob = await this.octokit.request( + "POST /repos/{owner}/{repo}/git/blobs", + { + ...this.getBasePayload(), + content: text, + encoding: "utf-8", + }, + ); + + return { + path: `${NOTE_PATH_BASE}${normalizePath(file.getPath())}`, + mode: "100644", + type: "blob", + sha: blob.data.sha, + }; + } catch (error) { + logger.error(error); + } + }); + + const treeAssetPromises = files + .flatMap((x) => x.compiledFile[1].images) + .map(async (asset) => { + try { + const blob = await this.octokit.request( + "POST /repos/{owner}/{repo}/git/blobs", + { + ...this.getBasePayload(), + content: asset.content, + encoding: "base64", + }, + ); + + return { + path: `${IMAGE_PATH_BASE}${normalizePath(asset.path)}`, + mode: "100644", + type: "blob", + sha: blob.data.sha, + }; + } catch (error) { + logger.error(error); + } + }); + treePromises.push(...treeAssetPromises); + + const treeList = await Promise.all(treePromises); + + //Filter away undefined values + const tree = treeList.filter((x) => x !== undefined) as { + path?: string | undefined; + mode?: + | "100644" + | "100755" + | "040000" + | "160000" + | "120000" + | undefined; + type?: "tree" | "blob" | "commit" | undefined; + sha?: string | null | undefined; + content?: string | undefined; + }[]; + + const newTree = await this.octokit.request( + "POST /repos/{owner}/{repo}/git/trees", + { + ...this.getBasePayload(), + base_tree: baseTreeSha, + tree, + }, + ); + + const commitMessage = "Published multiple files"; + + const newCommit = await this.octokit.request( + "POST /repos/{owner}/{repo}/git/commits", + { + ...this.getBasePayload(), + message: commitMessage, + tree: newTree.data.sha, + parents: [latestCommitSha], + }, + ); + + const defaultBranch = (await repoDataPromise).data.default_branch; + + await this.octokit.request( + "PATCH /repos/{owner}/{repo}/git/refs/heads/{branch}", + { + ...this.getBasePayload(), + branch: defaultBranch, + sha: newCommit.data.sha, + }, + ); + } + async getRepositoryInfo() { const repoInfo = await this.octokit .request("GET /repos/{owner}/{repo}", { diff --git a/src/test/snapshot/snapshot.md b/src/test/snapshot/snapshot.md index d25546d..da1293d 100644 --- a/src/test/snapshot/snapshot.md +++ b/src/test/snapshot/snapshot.md @@ -343,6 +343,55 @@ this is just text i guess --- +========== +015 Code blocks.md +========== +--- +{"dg-publish":true,"permalink":"/015-code-blocks/"} +--- + +These codeblocks should not be modified upon publish. + +Sample 1 +```jinja2 +{{ highlight_text }}{% if highlight_location and highlight_location_url %} ([via]({{highlight_location_url}})){% elif highlight_location %} ({{highlight_location}}){% endif %} ^rwhi{{highlight_id}} +{% if highlight_note %} +{{ highlight_note }} ^rwhi{{highlight_id}}-note +{% endif %} +``` + +Sample 2 +```md +In medieval Latin a florilegium (plural florilegia) was a compilation of excerpts from other writings. + The word is from the Latin flos (flower) and legere (to gather): literally a gathering of flowers, or collection of fine extracts from the body of a larger work. ([via](https://en.wikipedia.org/wiki/Florilegium)) ^rwhi724352030 +``` + +Sample 3 +``` +This codeblock has a transclusion syntax in it. +Check it out: +
+ + + + +[[002 Hidden page]] + +[[003 Non published page]] + +[[000 Home| Aliased link to home]] + +[[000 Home | Link containing whitespace which works in obsidian but doesn't in garden :) - yes, this could be a ticket but lo and behold]] + + + +
+ +``` + +And for sanity, here's some block references outside of code blocks: foobar +{ #test-123} + ========== E Embeds/E02 PNG published.md ========== @@ -715,7 +764,7 @@ P Plugins/PD Dataview/PD3 Inline JS queries.md 3 -106 +108

A paragraph

/img/user/A Assets/travolta.png @@ -730,7 +779,7 @@ P Plugins/PD Dataview/PD4 DataviewJs queries.md

Header 2

PD4 DataviewJs queries

-
name6link
006 Custom title006 Custom title
005 Custom filters005 Custom filters
007 Custom permalink007 Custom permalink
011 Custom updatedAt011 Custom updatedAt
013 Custom path013 Custom path
014 Customer path and permalink014 Customer path and permalink
+
name6link
005 Custom filters005 Custom filters
006 Custom title006 Custom title
007 Custom permalink007 Custom permalink
011 Custom updatedAt011 Custom updatedAt
013 Custom path013 Custom path
014 Customer path and permalink014 Customer path and permalink
/img/user/A Assets/travolta.png diff --git a/src/views/PublicationCenter/PublicationCenter.svelte b/src/views/PublicationCenter/PublicationCenter.svelte index b34fe6a..621564f 100644 --- a/src/views/PublicationCenter/PublicationCenter.svelte +++ b/src/views/PublicationCenter/PublicationCenter.svelte @@ -184,45 +184,29 @@ showPublishingView = true; - for (const note of changedToPublish.concat(unpublishedToPublish)) { - processingPaths.push(note.getPath()); - let isPublished = await publisher.publish(note); + const allNotesToPublish = unpublishedToPublish.concat(changedToPublish); - processingPaths = processingPaths.filter( - (path) => path !== note.getPath(), - ); - - if (isPublished) { - publishedPaths = [...publishedPaths, note.getPath()]; - } else { - failedPublish = [...failedPublish, note.getPath()]; - } - } - - for (const path of [...notesToDelete, ...imagesToDelete]) { - processingPaths.push(path); - const isNote = path.endsWith(".md"); - let isDeleted: boolean; + processingPaths = [...allNotesToPublish.map((note) => note.getPath())]; + await publisher.publishBatch(allNotesToPublish); - if (isNote) { - const sha = publishStatus.deletedNotePaths.find( - (p) => p.path === path, - )?.sha; - - isDeleted = await publisher.deleteNote(path, sha); - } else { - // TODO: remove with sha - isDeleted = await publisher.deleteImage(path); - } + publishedPaths = [...processingPaths]; + processingPaths = []; + for (const path of notesToDelete) { + processingPaths = [...processingPaths, path]; + await publisher.deleteNote(path); processingPaths = processingPaths.filter((p) => p !== path); + publishedPaths = [...publishedPaths, path]; + } - if (isDeleted) { - publishedPaths = [...publishedPaths, path]; - } else { - failedPublish = [...failedPublish, path]; - } + for (const path of imagesToDelete) { + processingPaths = [...processingPaths, path]; + await publisher.deleteImage(path); + processingPaths = processingPaths.filter((p) => p !== path); + publishedPaths = [...publishedPaths, path]; } + publishedPaths = [...publishedPaths, ...processingPaths]; + processingPaths = []; }; const emptyNode: TreeNode = { diff --git a/src/views/PublishStatusBar.ts b/src/views/PublishStatusBar.ts index ae99177..07ac8ec 100644 --- a/src/views/PublishStatusBar.ts +++ b/src/views/PublishStatusBar.ts @@ -16,6 +16,11 @@ export class PublishStatusBar { }); } + incrementMultiple(increments: number) { + this.counter += increments; + + this.status.innerText = `⌛Publishing files: ${this.counter}/${this.numberOfNotesToPublish}`; + } increment() { this.status.innerText = `⌛Publishing files: ${++this.counter}/${ this.numberOfNotesToPublish diff --git a/versions.json b/versions.json index 9019792..7caf5ac 100644 --- a/versions.json +++ b/versions.json @@ -1,4 +1,7 @@ { + "2.57.2": "0.12.0", + "2.57.1": "0.12.0", + "2.57.0": "0.12.0", "2.56.2": "0.12.0", "2.56.1": "0.12.0", "2.56.0": "0.12.0",