working builder for my blog

.gitignore

@@ -1,2 +1,3 @@
 target
-node_modules
+node_modules
+assets/syntax_cache.packdump

Cargo.lock

@@ -204,6 +204,16 @@ dependencies = [
  "regex",
 ]
 
+[[package]]
+name = "file-exists-macro"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6d89c0a483302ef815dfa9f8615b9fc98946eb7345b11c59d9fe7381e0ae2a95"
+dependencies = [
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "flate2"
 version = "1.1.0"
@@ -290,6 +300,7 @@ name = "mdsvexrs"
 version = "0.1.0"
 dependencies = [
  "clap",
+ "file-exists-macro",
  "itertools",
  "markdown",
  "regex",
@@ -393,9 +404,9 @@ dependencies = [
 
 [[package]]
 name = "quote"
-version = "1.0.39"
+version = "1.0.40"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c1f1914ce909e1658d9907913b4b91947430c7d9be598b15a1912935b8c04801"
+checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d"
 dependencies = [
  "proc-macro2",
 ]
@@ -503,9 +514,9 @@ checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
 
 [[package]]
 name = "syn"
-version = "2.0.99"
+version = "2.0.104"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e02e925281e18ffd9d640e234264753c43edc62d64b2d4cf898f1bc5e75f3fc2"
+checksum = "17b6f705963418cdb9927482fa304bc562ece2fdd4f616084c50b7023b435a40"
 dependencies = [
  "proc-macro2",
  "quote",

Cargo.toml

@@ -6,7 +6,7 @@ edition = "2021"
 [features]
 fancy = ["syntect/default-fancy"]
 onig = ["syntect/default-onig"]
-default = ["onig"]
+default = ["onig", "fancy"]
 
 [dependencies]
 markdown = { version = "1.0.0-alpha.21", features = ["serde"]}
@@ -16,4 +16,8 @@ serde_yaml = "0.9.34"
 serde = { version = "1.0.215", features = ["derive"] }
 regex = "1.11.1"
 clap = { version = "4.5.21", features = ["derive"] }
-syntect = { version = "5.0", default-features = false }
+syntect = { version = "5.0", default-features = false }
+file-exists-macro = "0.1"
+
+[build-dependencies]
+syntect = { version = "5.0", default-features = false, features = ["dump-create"] }

README.md

@@ -4,4 +4,8 @@ A faster markdown preprocessor for svelte. Compiles `.md` files into `.svelte` w
 
 Note that this, like the original MDSvex, trusts it's input and doesn't escape HTML or script files.
 
-This version is not yet tested and published.
+This version is not yet tested and published.
+
+Note that not all svelte syntax is supported yet. Notably, only HTML-like content is handled. If you get invalid syntax, try moving it into a component and just referencing that component. Templates, Ifs etc are not supported.
+
+Not all languages may be highlighted as syntect doesn't include support for all languages. Sublime syntax is supported and can be added on `Context.syntax_set`, but WASM don't have a way to set it - make an issue to embed the language instead.

build.rs

@@ -0,0 +1,10 @@
+use syntect::{dumps::dump_to_uncompressed_file, parsing::SyntaxSet};
+
+fn main() {
+    println!("cargo:rerun-if-changed=assets");
+    let mut ss = SyntaxSet::load_defaults_newlines().into_builder();
+    ss.add_from_folder("assets", true).unwrap();
+    let ss = ss.build();
+    
+    dump_to_uncompressed_file(&ss, "target/syntax_cache.packdump").expect("Failed to dump syntax cache");
+}

packages/vite/index.d.ts

@@ -1,5 +1,5 @@
 
-interface Options {
+export interface Options {
     layout: string;
 }
 
@@ -8,4 +8,4 @@ interface Plugin {
     markup: (opts: { content: string, filename: string }) => { code: string } | undefined;
 }
 
-export function mdsvexrs(options): Plugin;
+export function mdsvexrs(options: Options): Plugin;

packages/vite/index.js

@@ -1,4 +1,4 @@
-import { render } from "mdsvexrs-wasm"
+import * as wasm from "mdsvexrs-wasm"
 
 export function mdsvexrs(options) {
     return {
@@ -7,7 +7,7 @@ export function mdsvexrs(options) {
             if(!filename || !filename.endsWith('.md')) return
 
             return {
-                code: render(content, options)
+                code: wasm.render(content, options.layout)
             }
         }
     }

packages/vite/package.json

@@ -3,11 +3,12 @@
   "version": "0.1.0",
   "description": "",
   "main": "index.js",
+  "types": "index.d.ts",
   "scripts": {
     "test": "echo \"Error: no test specified\" && exit 1"
   },
   "dependencies": {
-    "mdsvexrs-wasm": "0.1.0"
+    "mdsvexrs-wasm": "file:../wasm/pkg"
   },
   "keywords": [],
   "author": "",

packages/wasm/Cargo.lock

@@ -201,6 +201,16 @@ dependencies = [
  "regex",
 ]
 
+[[package]]
+name = "file-exists-macro"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6d89c0a483302ef815dfa9f8615b9fc98946eb7345b11c59d9fe7381e0ae2a95"
+dependencies = [
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "flate2"
 version = "1.1.0"
@@ -287,6 +297,7 @@ name = "mdsvexrs"
 version = "0.1.0"
 dependencies = [
  "clap",
+ "file-exists-macro",
  "itertools",
  "markdown",
  "regex",
@@ -370,9 +381,9 @@ dependencies = [
 
 [[package]]
 name = "quote"
-version = "1.0.39"
+version = "1.0.40"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c1f1914ce909e1658d9907913b4b91947430c7d9be598b15a1912935b8c04801"
+checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d"
 dependencies = [
  "proc-macro2",
 ]
@@ -480,9 +491,9 @@ checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
 
 [[package]]
 name = "syn"
-version = "2.0.99"
+version = "2.0.104"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e02e925281e18ffd9d640e234264753c43edc62d64b2d4cf898f1bc5e75f3fc2"
+checksum = "17b6f705963418cdb9927482fa304bc562ece2fdd4f616084c50b7023b435a40"
 dependencies = [
  "proc-macro2",
  "quote",

packages/wasm/package.json

@@ -0,0 +1,6 @@
+{
+    "scripts": {
+        "build": "wasm-pack build --target nodejs"
+    },
+    "packageManager": "pnpm@9.5.0+sha1.8c155dc114e1689d18937974f6571e0ceee66f1d"
+}

packages/wasm/src/lib.rs

@@ -5,30 +5,36 @@ use mdsvexrs::Context;
 #[global_allocator]
 static ALLOC: wee_alloc::WeeAlloc = wee_alloc::WeeAlloc::INIT;
 
-#[wasm_bindgen]
-pub struct Options {
-    layout: String,
-    // path: String,
-}
+// #[wasm_bindgen]
+// pub struct Options {
+//     layout: String,
+//     // path: String,
+// }
+
+// #[wasm_bindgen]
+// impl Options {
+//     #[wasm_bindgen(getter)]
+//     pub fn layout(&self) -> String {
+//         self.layout.clone()
+//     }
+
+//     #[wasm_bindgen(setter)]
+//     pub fn set_layout(&mut self, layout: String) {
+//         self.layout = layout;
+//     }
+// }
+
+// #[wasm_bindgen]
+// pub fn get_default_options() -> Options {
+//     Options {
+//         layout: String::new(),
+//     }
+// }
 
 #[wasm_bindgen]
-impl Options {
-    #[wasm_bindgen(getter)]
-    pub fn layout(&self) -> String {
-        self.layout.clone()
-    }
-
-    #[wasm_bindgen(setter)]
-    pub fn set_layout(&mut self, layout: String) {
-        self.layout = layout;
-    }
-}
-
-
-#[wasm_bindgen]
-pub fn render(contents: &str, options: Options) -> String {
+pub fn render(contents: &str, layout: &str) -> String {
     Context::new(mdsvexrs::MdsvexrsOptions {
-        layout: options.layout.to_string(),
+        layout: layout.to_string(),
         // path: options.path,
     })
     .convert(contents)

pnpm-workspace.yaml

@@ -0,0 +1,3 @@
+packages:
+  - packages/vite
+  - packages/wasm/pkg

src/lib.rs

@@ -1,6 +1,6 @@
 use std::{
     sync::LazyLock,
-    time::{Duration, Instant},
+    time::{Duration/*, Instant*/},
 };
 
 use itertools::Itertools;
@@ -18,11 +18,7 @@ use markdown::{
 use serde::Serialize;
 use serde_json::Value;
 use syntect::{
-    easy::HighlightLines,
-    highlighting::ThemeSet,
-    html::{append_highlighted_html_for_styled_line, IncludeBackground},
-    parsing::SyntaxSet,
-    util::LinesWithEndings,
+    dumps::from_uncompressed_data, easy::HighlightLines, highlighting::ThemeSet, html::{append_highlighted_html_for_styled_line, IncludeBackground}, parsing::{SyntaxDefinition, SyntaxSet}, util::LinesWithEndings
 };
 
 #[derive(Debug)]
@@ -174,7 +170,7 @@ impl ToHtml for InlineCode {
 
 impl ToHtml for InlineMath {
     fn to_html(&self, _ctx: &mut Context) -> ToHtmlResult {
-        todo!()
+        ToHtmlResult::new(self.value.clone(), false)
     }
 }
 
@@ -542,13 +538,13 @@ pub struct Context {
     pub options: MdsvexrsOptions,
     pub titles: Vec<Title>,
 
-    syntax_set: SyntaxSet,
-    theme_set: ThemeSet,
+    pub syntax_set: SyntaxSet,
+    pub theme_set: ThemeSet,
 
-    pub(crate) highlight_times: Duration,
-    pub(crate) parse_time: Duration,
-    pub(crate) visit_time: Duration,
-    pub(crate) convert_time: Duration,
+    // pub(crate) highlight_times: Duration,
+    // pub(crate) parse_time: Duration,
+    // pub(crate) visit_time: Duration,
+    // pub(crate) convert_time: Duration,
 }
 
 #[derive(Serialize)]
@@ -567,7 +563,7 @@ pub struct MdsvexrsOptions {
 impl Context {
     pub fn new(options: MdsvexrsOptions) -> Self {
 
-        let syntax_set = SyntaxSet::load_defaults_newlines();
+        let syntax_set: SyntaxSet = from_uncompressed_data(include_bytes!("../target/syntax_cache.packdump")).unwrap();
         let theme_set = ThemeSet::load_defaults();
 
         Context {
@@ -580,16 +576,17 @@ impl Context {
             default_lang: None,
             script: None,
             options,
-            highlight_times: Duration::ZERO,
-            parse_time: Duration::ZERO,
-            visit_time: Duration::ZERO,
-            convert_time: Duration::ZERO,
+            // highlight_times: Duration::ZERO,
+            // parse_time: Duration::ZERO,
+            // visit_time: Duration::ZERO,
+            // convert_time: Duration::ZERO,
         }
     }
 
     fn highlight(&mut self, code: HighlightRequest) -> String {
-        let theme = &self.theme_set.themes["InspiredGitHub"];
-        let start = Instant::now();
+        let theme = &self.theme_set.themes["base16-ocean.dark"];
+        // #[cfg(not(target_arch = "wasm32"))]
+        // let start = Instant::now();
 
         let mut lang = &code.lang;
         if lang.is_empty() {
@@ -603,7 +600,7 @@ impl Context {
         let syntax = match syntax {
             Some(t) => t,
             None => {
-                return format!("<pre><code>{}</code></pre>", html_encode(&code.code));
+                return format!("<pre><code lang=\"{}\">{}</code></pre>", html_encode(lang), html_encode(&code.code));
             }
         };
         let mut highlighter = HighlightLines::new(syntax, theme);
@@ -615,7 +612,7 @@ impl Context {
                 let regions = highlighter
                     .highlight_line(&code.code, &self.syntax_set)
                     .unwrap();
-                string += "<code>";
+                string += &format!("<code lang=\"{}\">", html_encode(lang));
                 append_highlighted_html_for_styled_line(
                     &regions[..],
                     IncludeBackground::No,
@@ -625,7 +622,7 @@ impl Context {
                 string += "</code>";
             }
             false => {
-                string += "<pre>\n<code>";
+                string += &format!("<pre><code lang=\"{}\">", html_encode(lang));
                 for line in LinesWithEndings::from(&code.code) {
                     let regions = highlighter.highlight_line(line, &self.syntax_set).unwrap();
                     append_highlighted_html_for_styled_line(
@@ -639,7 +636,9 @@ impl Context {
             }
         };
 
-        self.highlight_times += start.elapsed();
+        // #[cfg(not(target_arch = "wasm32"))] {
+        //     self.highlight_times += start.elapsed();
+        // }
         string
     }
 
@@ -648,12 +647,18 @@ impl Context {
     }
 
     pub fn convert(&mut self, input: &str) -> String {
-        let start = Instant::now();
+        // #[cfg(not(target_arch = "wasm32"))]
+        // let start = Instant::now();
         let ast = markdown::to_mdast(input, &DEFAULT_MD_OPTIONS).unwrap();
-        self.parse_time = start.elapsed();
-        let start = Instant::now();
+        // #[cfg(not(target_arch = "wasm32"))] {
+        //     self.parse_time = start.elapsed();
+        // }
+        // #[cfg(not(target_arch = "wasm32"))]
+        // let start = Instant::now();
         ast.visit(self);
-        self.visit_time = start.elapsed();
+        // #[cfg(not(target_arch = "wasm32"))] {
+        //     self.visit_time = start.elapsed();
+        // }
 
         if let Some(yaml) = &self.yaml {
             if let Some(val) = yaml.get("defaultLang") {
@@ -661,10 +666,13 @@ impl Context {
             }
         }
 
-        let start = Instant::now();
+        // #[cfg(not(target_arch = "wasm32"))]
+        // let start = Instant::now();
         let res = ast.to_html(self);
         let html = finish(res);
-        self.convert_time = start.elapsed();
+        // #[cfg(not(target_arch = "wasm32"))] {
+        //     self.convert_time = start.elapsed();
+        // }
 
         if let Some(yaml) = &mut self.yaml {
             yaml.insert(
@@ -695,17 +703,22 @@ impl Context {
         format!(
             "<script context=\"module\">export const metadata = {frontmatter}</script>
 {script}
-<MDXLayout {{...metadata}}>
+<MDXLayout {{...metadata}} {{...$$restProps}}>
 {html}
 </MDXLayout>"
         )
     }
 
+    // #[cfg(not(target_arch = "wasm32"))]
+    // pub fn print_timings(&self) {
+    //     println!("Parse: {:?}", self.parse_time);
+    //     println!("Visit: {:?}", self.visit_time);
+    //     println!("Convert: {:?}", self.convert_time);
+    //     println!("Highlight: {:?}", self.highlight_times);
+    // }
+    // #[cfg(target_arch = "wasm32")]
     pub fn print_timings(&self) {
-        println!("Parse: {:?}", self.parse_time);
-        println!("Visit: {:?}", self.visit_time);
-        println!("Convert: {:?}", self.convert_time);
-        println!("Highlight: {:?}", self.highlight_times);
+        println!("wasm timings are not available");
     }
 }
 

src/main.rs

@@ -1,9 +1,10 @@
 use std::io::{stdin, Read};
 
-use clap::Parser;
+use clap::{Parser, Subcommand};
 use mdsvexrs::Context;
 
 #[derive(Parser)]
+#[command(version, about, long_about = None)]
 struct Args {
     #[arg(short, long)]
     layout: String,
@@ -13,8 +14,10 @@ struct Args {
     timings: bool,
 }
 
+
 fn main() {
     let args = Args::parse();
+
     let mut ctx = Context::new(mdsvexrs::MdsvexrsOptions {
         layout: args.layout,
         // path: args.path,