From 114c584bd6102ccd285f2a72170d6f2be18e8b80 Mon Sep 17 00:00:00 2001
From: oliverpool <3864879+oliverpool@users.noreply.github.com>
Date: Mon, 7 Oct 2024 00:05:36 +0200
Subject: [PATCH] Add Typst Lexer (#1007)

---
 README.md                      |   2 +-
 lexers/embedded/typst.xml      | 108 ++++++++++++++++++++
 lexers/testdata/typst.actual   |  22 +++++
 lexers/testdata/typst.expected | 176 +++++++++++++++++++++++++++++++++
 4 files changed, 307 insertions(+), 1 deletion(-)
 create mode 100644 lexers/embedded/typst.xml
 create mode 100644 lexers/testdata/typst.actual
 create mode 100644 lexers/testdata/typst.expected

diff --git a/README.md b/README.md
index bbe7f3639..34a7d15eb 100644
--- a/README.md
+++ b/README.md
@@ -52,7 +52,7 @@ translators for Pygments lexers and styles.
 |   Q    | QBasic, QML                                                                                                                                                                                                                                         |
 |   R    | R, Racket, Ragel, Raku, react, ReasonML, reg, Rego, reStructuredText, Rexx, RPMSpec, Ruby, Rust                                                                                                                                                     |
 |   S    | SAS, Sass, Scala, Scheme, Scilab, SCSS, Sed, Sieve, Smali, Smalltalk, Smarty, SNBT, Snobol, Solidity, SourcePawn, SPARQL, SQL, SquidConf, Standard ML, stas, Stylus, Svelte, Swift, SYSTEMD, systemverilog                                                |
-|   T    | TableGen, Tal, TASM, Tcl, Tcsh, Termcap, Terminfo, Terraform, TeX, Thrift, TOML, TradingView, Transact-SQL, Turing, Turtle, Twig, TypeScript, TypoScript, TypoScriptCssData, TypoScriptHtmlData                                                     |
+|   T    | TableGen, Tal, TASM, Tcl, Tcsh, Termcap, Terminfo, Terraform, TeX, Thrift, TOML, TradingView, Transact-SQL, Turing, Turtle, Twig, TypeScript, TypoScript, TypoScriptCssData, TypoScriptHtmlData, Typst                                              |
 |   V    | V, V shell, Vala, VB.net, verilog, VHDL, VHS, VimL, vue                                                                                                                                                                                             |
 |   W    | WDTE, WebGPU Shading Language, Whiley                                                                                                                                                                                                               |
 |   X    | XML, Xorg                                                                                                                                                                                                                                           |
diff --git a/lexers/embedded/typst.xml b/lexers/embedded/typst.xml
new file mode 100644
index 000000000..330dc40fa
--- /dev/null
+++ b/lexers/embedded/typst.xml
@@ -0,0 +1,108 @@
+
+<lexer>
+  <config>
+    <name>Typst</name>
+    <alias>typst</alias>
+    <filename>*.typ</filename>
+    <mime_type>text/x-typst</mime_type>
+  </config>
+  <rules>
+    <state name="root">
+      <rule><include state="markup"/></rule>
+    </state>
+    <state name="into_code">
+      <rule pattern="(\#let|\#set|\#show)\b"><token type="KeywordDeclaration"/><push state="inline_code"/></rule>
+      <rule pattern="(\#import|\#include)\b"><token type="KeywordNamespace"/><push state="inline_code"/></rule>
+      <rule pattern="(\#if|\#for|\#while|\#export)\b"><token type="KeywordReserved"/><push state="inline_code"/></rule>
+      <rule pattern="#\{"><token type="Punctuation"/><push state="code"/></rule>
+      <rule pattern="#\("><token type="Punctuation"/><push state="code"/></rule>
+      <rule pattern="(#[a-zA-Z_][a-zA-Z0-9_-]*)(\[)"><bygroups><token type="NameFunction"/><token type="Punctuation"/></bygroups><push state="markup"/></rule>
+      <rule pattern="(#[a-zA-Z_][a-zA-Z0-9_-]*)(\()"><bygroups><token type="NameFunction"/><token type="Punctuation"/></bygroups><push state="code"/></rule>
+      <rule pattern="(\#true|\#false|\#none|\#auto)\b"><token type="KeywordConstant"/></rule>
+      <rule pattern="#[a-zA-Z_][a-zA-Z0-9_]*"><token type="NameVariable"/></rule>
+      <rule pattern="#0x[0-9a-fA-F]+"><token type="LiteralNumberHex"/></rule>
+      <rule pattern="#0b[01]+"><token type="LiteralNumberBin"/></rule>
+      <rule pattern="#0o[0-7]+"><token type="LiteralNumberOct"/></rule>
+      <rule pattern="#[0-9]+[\.e][0-9]+"><token type="LiteralNumberFloat"/></rule>
+      <rule pattern="#[0-9]+"><token type="LiteralNumberInteger"/></rule>
+    </state>
+    <state name="markup">
+      <rule><include state="comment"/></rule>
+      <rule pattern="^\s*=+.*$"><token type="GenericHeading"/></rule>
+      <rule pattern="[*][^*]*[*]"><token type="GenericStrong"/></rule>
+      <rule pattern="_[^_]*_"><token type="GenericEmph"/></rule>
+      <rule pattern="\$"><token type="Punctuation"/><push state="math"/></rule>
+      <rule pattern="`[^`]*`"><token type="LiteralStringBacktick"/></rule>
+      <rule pattern="^(\s*)(-)(\s+)"><bygroups><token type="TextWhitespace"/><token type="Punctuation"/><token type="TextWhitespace"/></bygroups></rule>
+      <rule pattern="^(\s*)(\+)(\s+)"><bygroups><token type="TextWhitespace"/><token type="Punctuation"/><token type="TextWhitespace"/></bygroups></rule>
+      <rule pattern="^(\s*)([0-9]+\.)"><bygroups><token type="TextWhitespace"/><token type="Punctuation"/></bygroups></rule>
+      <rule pattern="^(\s*)(/)(\s+)([^:]+)(:)"><bygroups><token type="TextWhitespace"/><token type="Punctuation"/><token type="TextWhitespace"/><token type="NameVariable"/><token type="Punctuation"/></bygroups></rule>
+      <rule pattern="&lt;[a-zA-Z_][a-zA-Z0-9_-]*&gt;"><token type="NameLabel"/></rule>
+      <rule pattern="@[a-zA-Z_][a-zA-Z0-9_-]*"><token type="NameLabel"/></rule>
+      <rule pattern="\\#"><token type="Text"/></rule>
+      <rule><include state="into_code"/></rule>
+      <rule pattern="```(?:.|\n)*?```"><token type="LiteralStringBacktick"/></rule>
+      <rule pattern="https?://[0-9a-zA-Z~/%#&amp;=\&#x27;,;.+?]*"><token type="GenericEmph"/></rule>
+      <rule pattern="(\-\-\-|\\|\~|\-\-|\.\.\.)\B"><token type="Punctuation"/></rule>
+      <rule pattern="\\\["><token type="Punctuation"/></rule>
+      <rule pattern="\\\]"><token type="Punctuation"/></rule>
+      <rule pattern="\["><token type="Punctuation"/><push/></rule>
+      <rule pattern="\]"><token type="Punctuation"/><pop depth="1"/></rule>
+      <rule pattern="[ \t]+\n?|\n"><token type="TextWhitespace"/></rule>
+      <rule pattern="((?![*_$`&lt;@\\#\] ]|https?://).)+"><token type="Text"/></rule>
+    </state>
+    <state name="math">
+      <rule><include state="comment"/></rule>
+      <rule pattern="(\\_|\\\^|\\\&amp;)"><token type="Text"/></rule>
+      <rule pattern="(_|\^|\&amp;|;)"><token type="Punctuation"/></rule>
+      <rule pattern="(\+|/|=|\[\||\|\]|\|\||\*|:=|::=|\.\.\.|&#x27;|\-|=:|!=|&gt;&gt;|&gt;=|&gt;&gt;&gt;|&lt;&lt;|&lt;=|&lt;&lt;&lt;|\-&gt;|\|\-&gt;|=&gt;|\|=&gt;|==&gt;|\-\-&gt;|\~\~&gt;|\~&gt;|&gt;\-&gt;|\-&gt;&gt;|&lt;\-|&lt;==|&lt;\-\-|&lt;\~\~|&lt;\~|&lt;\-&lt;|&lt;&lt;\-|&lt;\-&gt;|&lt;=&gt;|&lt;==&gt;|&lt;\-\-&gt;|&gt;|&lt;|\~|:|\|)"><token type="Operator"/></rule>
+      <rule pattern="\\"><token type="Punctuation"/></rule>
+      <rule pattern="\\\$"><token type="Punctuation"/></rule>
+      <rule pattern="\$"><token type="Punctuation"/><pop depth="1"/></rule>
+      <rule><include state="into_code"/></rule>
+      <rule pattern="([a-zA-Z][a-zA-Z0-9-]*)(\s*)(\()"><bygroups><token type="NameFunction"/><token type="TextWhitespace"/><token type="Punctuation"/></bygroups></rule>
+      <rule pattern="([a-zA-Z][a-zA-Z0-9-]*)(:)"><bygroups><token type="NameVariable"/><token type="Punctuation"/></bygroups></rule>
+      <rule pattern="([a-zA-Z][a-zA-Z0-9-]*)"><token type="NameVariable"/></rule>
+      <rule pattern="[0-9]+(\.[0-9]+)?"><token type="LiteralNumber"/></rule>
+      <rule pattern="\.{1,3}|\(|\)|,|\{|\}"><token type="Punctuation"/></rule>
+      <rule pattern="&quot;[^&quot;]*&quot;"><token type="LiteralStringDouble"/></rule>
+      <rule pattern="[ \t\n]+"><token type="TextWhitespace"/></rule>
+    </state>
+    <state name="comment">
+      <rule pattern="//.*$"><token type="CommentSingle"/></rule>
+      <rule pattern="/[*](.|\n)*?[*]/"><token type="CommentMultiline"/></rule>
+    </state>
+    <state name="code">
+      <rule><include state="comment"/></rule>
+      <rule pattern="\["><token type="Punctuation"/><push state="markup"/></rule>
+      <rule pattern="\(|\{"><token type="Punctuation"/><push state="code"/></rule>
+      <rule pattern="\)|\}"><token type="Punctuation"/><pop depth="1"/></rule>
+      <rule pattern="&quot;[^&quot;]*&quot;"><token type="LiteralStringDouble"/></rule>
+      <rule pattern=",|\.{1,2}"><token type="Punctuation"/></rule>
+      <rule pattern="="><token type="Operator"/></rule>
+      <rule pattern="(and|or|not)\b"><token type="OperatorWord"/></rule>
+      <rule pattern="=&gt;|&lt;=|==|!=|&gt;|&lt;|-=|\+=|\*=|/=|\+|-|\\|\*"><token type="Operator"/></rule>
+      <rule pattern="([a-zA-Z_][a-zA-Z0-9_-]*)(:)"><bygroups><token type="NameVariable"/><token type="Punctuation"/></bygroups></rule>
+      <rule pattern="([a-zA-Z_][a-zA-Z0-9_-]*)(\()"><bygroups><token type="NameFunction"/><token type="Punctuation"/></bygroups><push state="code"/></rule>
+      <rule pattern="(as|break|export|continue|else|for|if|in|return|while)\b"><token type="KeywordReserved"/></rule>
+      <rule pattern="(import|include)\b"><token type="KeywordNamespace"/></rule>
+      <rule pattern="(auto|none|true|false)\b"><token type="KeywordConstant"/></rule>
+      <rule pattern="([0-9.]+)(mm|pt|cm|in|em|fr|%)"><bygroups><token type="LiteralNumber"/><token type="KeywordReserved"/></bygroups></rule>
+      <rule pattern="0x[0-9a-fA-F]+"><token type="LiteralNumberHex"/></rule>
+      <rule pattern="0b[01]+"><token type="LiteralNumberBin"/></rule>
+      <rule pattern="0o[0-7]+"><token type="LiteralNumberOct"/></rule>
+      <rule pattern="[0-9]+[\.e][0-9]+"><token type="LiteralNumberFloat"/></rule>
+      <rule pattern="[0-9]+"><token type="LiteralNumberInteger"/></rule>
+      <rule pattern="(let|set|show)\b"><token type="KeywordDeclaration"/></rule>
+      <rule pattern="([a-zA-Z_][a-zA-Z0-9_-]*)"><token type="NameVariable"/></rule>
+      <rule pattern="[ \t\n]+"><token type="TextWhitespace"/></rule>
+      <rule pattern=":"><token type="Punctuation"/></rule>
+    </state>
+    <state name="inline_code">
+      <rule pattern=";\b"><token type="Punctuation"/><pop depth="1"/></rule>
+      <rule pattern="\n"><token type="TextWhitespace"/><pop depth="1"/></rule>
+      <rule><include state="code"/></rule>
+    </state>
+  </rules>
+</lexer>
+
diff --git a/lexers/testdata/typst.actual b/lexers/testdata/typst.actual
new file mode 100644
index 000000000..e8c797d56
--- /dev/null
+++ b/lexers/testdata/typst.actual
@@ -0,0 +1,22 @@
+#let song(
+  title: none,
+  title_index: none,
+  singer: none,
+  singer_index: none,
+  references: (),
+  line_color: rgb(0xd0, 0xd0, 0xd0),
+  header_display: (number, title, singer) => grid(
+    columns: (auto,1fr),
+    inset: 4pt,
+    align: horizon,
+    grid.cell(fill: rgb(0xd0, 0xd0, 0xd0),align: top, inset:(rest:4pt, /*right: 10pt*/), heading(outlined: false,str(number))),
+    [
+      #heading(title)
+      #if singer != none {
+        strong(singer) + "\n"
+      }
+    ],
+  ),
+  // comment
+  doc,
+)
diff --git a/lexers/testdata/typst.expected b/lexers/testdata/typst.expected
new file mode 100644
index 000000000..ede015ae1
--- /dev/null
+++ b/lexers/testdata/typst.expected
@@ -0,0 +1,176 @@
+[
+  {"type":"KeywordDeclaration","value":"#let"},
+  {"type":"TextWhitespace","value":" "},
+  {"type":"NameFunction","value":"song"},
+  {"type":"Punctuation","value":"("},
+  {"type":"TextWhitespace","value":"\n  "},
+  {"type":"NameVariable","value":"title"},
+  {"type":"Punctuation","value":":"},
+  {"type":"TextWhitespace","value":" "},
+  {"type":"KeywordConstant","value":"none"},
+  {"type":"Punctuation","value":","},
+  {"type":"TextWhitespace","value":"\n  "},
+  {"type":"NameVariable","value":"title_index"},
+  {"type":"Punctuation","value":":"},
+  {"type":"TextWhitespace","value":" "},
+  {"type":"KeywordConstant","value":"none"},
+  {"type":"Punctuation","value":","},
+  {"type":"TextWhitespace","value":"\n  "},
+  {"type":"NameVariable","value":"singer"},
+  {"type":"Punctuation","value":":"},
+  {"type":"TextWhitespace","value":" "},
+  {"type":"KeywordConstant","value":"none"},
+  {"type":"Punctuation","value":","},
+  {"type":"TextWhitespace","value":"\n  "},
+  {"type":"NameVariable","value":"singer_index"},
+  {"type":"Punctuation","value":":"},
+  {"type":"TextWhitespace","value":" "},
+  {"type":"KeywordConstant","value":"none"},
+  {"type":"Punctuation","value":","},
+  {"type":"TextWhitespace","value":"\n  "},
+  {"type":"NameVariable","value":"references"},
+  {"type":"Punctuation","value":":"},
+  {"type":"TextWhitespace","value":" "},
+  {"type":"Punctuation","value":"(),"},
+  {"type":"TextWhitespace","value":"\n  "},
+  {"type":"NameVariable","value":"line_color"},
+  {"type":"Punctuation","value":":"},
+  {"type":"TextWhitespace","value":" "},
+  {"type":"NameFunction","value":"rgb"},
+  {"type":"Punctuation","value":"("},
+  {"type":"LiteralNumberHex","value":"0xd0"},
+  {"type":"Punctuation","value":","},
+  {"type":"TextWhitespace","value":" "},
+  {"type":"LiteralNumberHex","value":"0xd0"},
+  {"type":"Punctuation","value":","},
+  {"type":"TextWhitespace","value":" "},
+  {"type":"LiteralNumberHex","value":"0xd0"},
+  {"type":"Punctuation","value":"),"},
+  {"type":"TextWhitespace","value":"\n  "},
+  {"type":"NameVariable","value":"header_display"},
+  {"type":"Punctuation","value":":"},
+  {"type":"TextWhitespace","value":" "},
+  {"type":"Punctuation","value":"("},
+  {"type":"NameVariable","value":"number"},
+  {"type":"Punctuation","value":","},
+  {"type":"TextWhitespace","value":" "},
+  {"type":"NameVariable","value":"title"},
+  {"type":"Punctuation","value":","},
+  {"type":"TextWhitespace","value":" "},
+  {"type":"NameVariable","value":"singer"},
+  {"type":"Punctuation","value":")"},
+  {"type":"TextWhitespace","value":" "},
+  {"type":"Operator","value":"=\u003e"},
+  {"type":"TextWhitespace","value":" "},
+  {"type":"NameFunction","value":"grid"},
+  {"type":"Punctuation","value":"("},
+  {"type":"TextWhitespace","value":"\n    "},
+  {"type":"NameVariable","value":"columns"},
+  {"type":"Punctuation","value":":"},
+  {"type":"TextWhitespace","value":" "},
+  {"type":"Punctuation","value":"("},
+  {"type":"KeywordConstant","value":"auto"},
+  {"type":"Punctuation","value":","},
+  {"type":"LiteralNumber","value":"1"},
+  {"type":"KeywordReserved","value":"fr"},
+  {"type":"Punctuation","value":"),"},
+  {"type":"TextWhitespace","value":"\n    "},
+  {"type":"NameVariable","value":"inset"},
+  {"type":"Punctuation","value":":"},
+  {"type":"TextWhitespace","value":" "},
+  {"type":"LiteralNumber","value":"4"},
+  {"type":"KeywordReserved","value":"pt"},
+  {"type":"Punctuation","value":","},
+  {"type":"TextWhitespace","value":"\n    "},
+  {"type":"NameVariable","value":"align"},
+  {"type":"Punctuation","value":":"},
+  {"type":"TextWhitespace","value":" "},
+  {"type":"NameVariable","value":"horizon"},
+  {"type":"Punctuation","value":","},
+  {"type":"TextWhitespace","value":"\n    "},
+  {"type":"NameVariable","value":"grid"},
+  {"type":"Punctuation","value":"."},
+  {"type":"NameFunction","value":"cell"},
+  {"type":"Punctuation","value":"("},
+  {"type":"NameVariable","value":"fill"},
+  {"type":"Punctuation","value":":"},
+  {"type":"TextWhitespace","value":" "},
+  {"type":"NameFunction","value":"rgb"},
+  {"type":"Punctuation","value":"("},
+  {"type":"LiteralNumberHex","value":"0xd0"},
+  {"type":"Punctuation","value":","},
+  {"type":"TextWhitespace","value":" "},
+  {"type":"LiteralNumberHex","value":"0xd0"},
+  {"type":"Punctuation","value":","},
+  {"type":"TextWhitespace","value":" "},
+  {"type":"LiteralNumberHex","value":"0xd0"},
+  {"type":"Punctuation","value":"),"},
+  {"type":"NameVariable","value":"align"},
+  {"type":"Punctuation","value":":"},
+  {"type":"TextWhitespace","value":" "},
+  {"type":"NameVariable","value":"top"},
+  {"type":"Punctuation","value":","},
+  {"type":"TextWhitespace","value":" "},
+  {"type":"NameVariable","value":"inset"},
+  {"type":"Punctuation","value":":("},
+  {"type":"NameVariable","value":"rest"},
+  {"type":"Punctuation","value":":"},
+  {"type":"LiteralNumber","value":"4"},
+  {"type":"KeywordReserved","value":"pt"},
+  {"type":"Punctuation","value":","},
+  {"type":"TextWhitespace","value":" "},
+  {"type":"CommentMultiline","value":"/*right: 10pt*/"},
+  {"type":"Punctuation","value":"),"},
+  {"type":"TextWhitespace","value":" "},
+  {"type":"NameFunction","value":"heading"},
+  {"type":"Punctuation","value":"("},
+  {"type":"NameVariable","value":"outlined"},
+  {"type":"Punctuation","value":":"},
+  {"type":"TextWhitespace","value":" "},
+  {"type":"KeywordConstant","value":"false"},
+  {"type":"Punctuation","value":","},
+  {"type":"NameFunction","value":"str"},
+  {"type":"Punctuation","value":"("},
+  {"type":"NameVariable","value":"number"},
+  {"type":"Punctuation","value":"))),"},
+  {"type":"TextWhitespace","value":"\n    "},
+  {"type":"Punctuation","value":"["},
+  {"type":"TextWhitespace","value":"\n      "},
+  {"type":"NameFunction","value":"#heading"},
+  {"type":"Punctuation","value":"("},
+  {"type":"NameVariable","value":"title"},
+  {"type":"Punctuation","value":")"},
+  {"type":"TextWhitespace","value":"\n      "},
+  {"type":"KeywordReserved","value":"#if"},
+  {"type":"TextWhitespace","value":" "},
+  {"type":"NameVariable","value":"singer"},
+  {"type":"TextWhitespace","value":" "},
+  {"type":"Operator","value":"!="},
+  {"type":"TextWhitespace","value":" "},
+  {"type":"KeywordConstant","value":"none"},
+  {"type":"TextWhitespace","value":" "},
+  {"type":"Punctuation","value":"{"},
+  {"type":"TextWhitespace","value":"\n        "},
+  {"type":"NameFunction","value":"strong"},
+  {"type":"Punctuation","value":"("},
+  {"type":"NameVariable","value":"singer"},
+  {"type":"Punctuation","value":")"},
+  {"type":"TextWhitespace","value":" "},
+  {"type":"Operator","value":"+"},
+  {"type":"TextWhitespace","value":" "},
+  {"type":"LiteralStringDouble","value":"\"\\n\""},
+  {"type":"TextWhitespace","value":"\n      "},
+  {"type":"Punctuation","value":"}"},
+  {"type":"TextWhitespace","value":"\n    "},
+  {"type":"Punctuation","value":"],"},
+  {"type":"TextWhitespace","value":"\n  "},
+  {"type":"Punctuation","value":"),"},
+  {"type":"TextWhitespace","value":"\n  "},
+  {"type":"CommentSingle","value":"// comment"},
+  {"type":"TextWhitespace","value":"\n  "},
+  {"type":"NameVariable","value":"doc"},
+  {"type":"Punctuation","value":","},
+  {"type":"TextWhitespace","value":"\n"},
+  {"type":"Punctuation","value":")"},
+  {"type":"TextWhitespace","value":"\n"}
+]