Parsing MyST Markdown with TypeScript

So I put together this editor/preview component for the mystmd.org page like I've said in previous posts (some way to crossref sections in other posts?, also this editor is slow as christmas with all this tex in here lmfao)

So here's the react component I put together for my site

theme/src/components/MultiFileMySTDemo.tsx

import { ProjectBrowser, SaveProjectModal, RenderSuccessPopup } from './ProjectModal';
import { FileExplorer } from './FileExplorer';
import { CreativeCommonsIcon, AttributionIcon, ORCIDIcon, ApacheIcon } from './Icons';
import { CodeEditor } from './CodeEditor';
import { FrontmatterBlock } from './FrontmatterBlock';
import React, { useEffect, useState, useRef } from 'react';
import classnames from 'classnames';
import { ArticleProvider, GridSystemProvider } from '@myst-theme/providers';
import { MyST } from 'myst-to-react';
import { EditorView, basicSetup } from 'codemirror';
import { EditorState } from '@codemirror/state';
import { markdown } from '@codemirror/lang-markdown';
import { yaml } from '@codemirror/lang-yaml';
import { css } from '@codemirror/lang-css';
import { lineNumbers, EditorView as EditorViewType } from '@codemirror/view';
import { oneDark } from '@codemirror/theme-one-dark';

/**
 * EXACTLY LIKE THE ORIGINAL - Real-time MyST parsing with debouncing
 * BUT now we combine multiple markdown files for parsing
 */
async function parseMarkdown(md: string) {
  // Keep the EXACT same parsing logic from CustomMySTDemo
  const { VFile } = await import('vfile');
  const { visit } = await import('unist-util-visit');
  const { unified } = await import('unified');
  const { mystParse } = await import('myst-parser');
  const {
    mathPlugin,
    footnotesPlugin,
    keysPlugin,
    htmlPlugin,
    reconstructHtmlPlugin,
    basicTransformationsPlugin,
    enumerateTargetsPlugin,
    resolveReferencesPlugin,
    WikiTransformer,
    GithubTransformer,
    DOITransformer,
    RRIDTransformer,
    RORTransformer,
    linksPlugin,
    ReferenceState,
    abbreviationPlugin,
    glossaryPlugin,
    joinGatesPlugin,
  } = await import('myst-transforms');
  const { mystToHtml } = await import('myst-to-html');
  const { buttonRole } = await import('myst-ext-button');
  const { cardDirective } = await import('myst-ext-card');
  const { gridDirectives } = await import('myst-ext-grid');
  const { tabDirectives } = await import('myst-ext-tabs');
  const { proofDirective } = await import('myst-ext-proof');
  const { exerciseDirectives } = await import('myst-ext-exercise');
  const { validatePageFrontmatter } = await import('myst-frontmatter');
  const { remove } = await import('unist-util-remove');
  const { load: yamlLoad } = await import('js-yaml');
  const { fileError, RuleId } = await import('myst-common');

  const vfile = new VFile();
  
  const parseMyst = (content: string) =>
    mystParse(content, {
      markdownit: { linkify: true },
      directives: [
        cardDirective,
        ...gridDirectives,
        ...tabDirectives,
        proofDirective,
        ...exerciseDirectives,
      ],
      roles: [buttonRole],
      vfile,
    });

  const mdast = parseMyst(md);
  
  const linkTransforms = [
    new WikiTransformer(),
    new GithubTransformer(),
    new DOITransformer(),
    new RRIDTransformer(),
    new RORTransformer(),
  ];

  // Frontmatter extraction (same as original)
  const firstParent = mdast.children[0]?.type === 'block' ? mdast.children[0] : mdast;
  const firstNode = firstParent.children?.[0];
  let frontmatterRaw = {};
  const firstIsYaml = firstNode?.type === 'code' && firstNode?.lang === 'yaml';
  if (firstIsYaml) {
    try {
      frontmatterRaw = (yamlLoad(firstNode.value) as Record<string, any>) || {};
      (firstNode as any).type = '__delete__';
    } catch (err) {
      fileError(vfile, 'Invalid YAML frontmatter', {
        note: (err as Error).message,
        ruleId: RuleId.frontmatterIsYaml,
      });
    }
  }
  
  const possibleNull = remove(mdast, '__delete__');
  if (possibleNull === null) {
    remove(mdast, { cascade: false }, '__delete__');
  }

  const validatedFrontmatter = validatePageFrontmatter(frontmatterRaw, {
    property: 'frontmatter',
    messages: {},
  });

  const references = {
    cite: { order: [], data: {} },
  };

  const state = new ReferenceState('', {
    frontmatter: validatedFrontmatter,
    vfile,
  });

  visit(mdast, (n) => {
    if (n.type === 'cite') {
      n.error = true;
    }
  });

  // Apply all transforms for full MyST functionality
  unified()
    .use(reconstructHtmlPlugin)
    .use(htmlPlugin)
    .use(basicTransformationsPlugin, { parser: parseMyst })
    .use(mathPlugin, { macros: validatedFrontmatter?.math ?? {} })
    .use(glossaryPlugin)
    .use(abbreviationPlugin, { abbreviations: validatedFrontmatter.abbreviations })
    .use(enumerateTargetsPlugin, { state })
    .use(linksPlugin, { transformers: linkTransforms })
    .use(footnotesPlugin)
    .use(joinGatesPlugin)
    .use(resolveReferencesPlugin, { state })
    .use(keysPlugin)
    .runSync(mdast as any, vfile);

  const html = mystToHtml(JSON.parse(JSON.stringify(mdast)));

  return { 
    mdast, 
    html, 
    frontmatter: validatedFrontmatter,
    references: { ...references, article: mdast }
  };
}

// Backend API function
async function renderProject(files: Record<string, string>) {
  const response = await fetch('http://localhost:8000/render', {
    method: 'POST',
    headers: { 'Content-Type': 'application/json' },
    body: JSON.stringify({ files }),
  });

  if (!response.ok) {
    const errorText = await response.text();
    throw new Error(`HTTP error! status: ${response.status}, message: ${errorText}`);
  }

  return await response.json();
}

// Save project function (imported from ProjectModal)
async function saveProject(name: string, description: string, files: Record<string, string>, projectId?: string) {
  const response = await fetch('http://localhost:8000/projects', {
    method: 'POST',
    headers: { 'Content-Type': 'application/json' },
    body: JSON.stringify({ id: projectId, name, description, files }),
  });

  if (!response.ok) {
    const errorText = await response.text();
    throw new Error(`HTTP error! status: ${response.status}, message: ${errorText}`);
  }

  return await response.json();
}

// MAIN COMPONENT - CLEAN STATE MANAGEMENT
export default function MultiFileMySTDemo({
  files: initialFiles,
  onFilesChange,
  className,
}: {
  files: Record<string, string>;
  onFilesChange: (files: Record<string, string>) => void;
  className?: string;
}) {
  // SIMPLE STATE - NO COMPLEX REFS OR FLAGS
  const [files, setFiles] = useState(initialFiles);
  const [activeFile, setActiveFile] = useState('index.md');
  const [currentProject, setCurrentProject] = useState<any>(null);
  
  // Add file explorer collapse state
  const [isFileExplorerCollapsed, setIsFileExplorerCollapsed] = useState(false);
  
  // UI state
  const [showSaveModal, setShowSaveModal] = useState(false);
  const [showProjectBrowser, setShowProjectBrowser] = useState(false);
  const [saveStatus, setSaveStatus] = useState<'idle' | 'saving' | 'saved' | 'error'>('idle');

  // MyST parsing state
  const [mdast, setMdast] = useState<any>(null);
  const [frontmatter, setFrontmatter] = useState<any>({});
  const [references, setReferences] = useState<any>({});
  const [lastValidParseResult, setLastValidParseResult] = useState<{
    mdast: any;
    frontmatter: any;
    references: any;
    content: string;
  } | null>(null);
  const [isParsingRef, setIsParsingRef] = useState<{ current: boolean }>({ current: false });

  // Render state
  const [renderResult, setRenderResult] = useState<any>(null);
  const [isRendering, setIsRendering] = useState(false);
  const [renderError, setRenderError] = useState<string>('');
  const [showRenderPopup, setShowRenderPopup] = useState(false);

  // Keyboard shortcut handler for Ctrl+S
  useEffect(() => {
    const handleKeyDown = (event: KeyboardEvent) => {
      if ((event.ctrlKey || event.metaKey) && event.key === 's') {
        event.preventDefault();
        handleQuickSave();
      }
    };

    document.addEventListener('keydown', handleKeyDown);
    return () => document.removeEventListener('keydown', handleKeyDown);
  }, [files, currentProject]);

  // Quick save function (auto-save without modal)
  const handleQuickSave = async () => {
    if (saveStatus === 'saving') return;

    setSaveStatus('saving');

    try {
      let projectName = currentProject?.name;
      let projectDescription = currentProject?.description || '';
      let projectId = currentProject?.id;

      if (!currentProject) {
        const timestamp = new Date().toLocaleString();
        projectName = `MyST Project - ${timestamp}`;
        projectDescription = 'Auto-saved project';
      }

      const result = await saveProject(projectName, projectDescription, files, projectId);
      setCurrentProject(result);
      setSaveStatus('saved');

      setTimeout(() => setSaveStatus('idle'), 2000);
    } catch (error) {
      console.error('Quick save failed:', error);
      setSaveStatus('error');
      setTimeout(() => setSaveStatus('idle'), 3000);
    }
  };

  // Helper function
  const isMarkdownFile = (filename: string) => filename.endsWith('.md');
  
  // Get the current file content if it's markdown
  const activeFileContent = files[activeFile] || '';
  const shouldShowPreview = isMarkdownFile(activeFile);

  // Parse ONLY the active markdown file content - with graceful fallback
  useEffect(() => {
    if (!shouldShowPreview || !activeFileContent.trim()) {
      setMdast(null);
      setFrontmatter({});
      setReferences({});
      setLastValidParseResult(null);
      return;
    }

    const timeoutId = setTimeout(() => {
      if (isParsingRef.current) return;
      
      isParsingRef.current = true;
      parseMarkdown(activeFileContent)
        .then((result) => {
          setMdast(result.mdast);
          setFrontmatter(result.frontmatter);
          setReferences(result.references);
          setLastValidParseResult({
            mdast: result.mdast,
            frontmatter: result.frontmatter,
            references: result.references,
            content: activeFileContent
          });
        })
        .catch((error) => {
          console.error('Parse error:', error);
          if (lastValidParseResult && lastValidParseResult.content !== activeFileContent) {
            // Keep showing last valid result
          } else {
            setMdast(null);
            setFrontmatter({});
            setReferences({});
          }
        })
        .finally(() => {
          isParsingRef.current = false;
        });
    }, 300);

    return () => clearTimeout(timeoutId);
  }, [activeFileContent, shouldShowPreview, activeFile]);

  // Reset last valid state when switching files
  useEffect(() => {
    setLastValidParseResult(null);
  }, [activeFile]);

  const handleFileChange = (filename: string, content: string) => {
    const newFiles = { ...files, [filename]: content };
    setFiles(newFiles);
    onFilesChange(newFiles);
  };

  const handleFileAdd = (filename: string) => {
    const newFiles = { ...files, [filename]: '' };
    setFiles(newFiles);
    onFilesChange(newFiles);
  };

  const handleFileDelete = (filename: string) => {
    const newFiles = { ...files };
    delete newFiles[filename];
    setFiles(newFiles);
    onFilesChange(newFiles);
    
    if (activeFile === filename) {
      setActiveFile('index.md');
    }
  };

  const handleRender = async () => {
    setIsRendering(true);
    setRenderError('');
    
    try {
      const result = await renderProject(files);
      setRenderResult(result);
      if (result.status === 'success') {
        setShowRenderPopup(true);
      }
    } catch (error) {
      setRenderError(error instanceof Error ? error.message : 'Unknown error');
    } finally {
      setIsRendering(false);
    }
  };

  const handleProjectSave = (project: any) => {
    setCurrentProject(project);
  };

  // CLEAN PROJECT LOADING - FORCE EVERYTHING
  const handleProjectLoad = (project: any) => {
    console.log('Loading project:', project);
    console.log('Current files before:', Object.keys(files));
    
    // Get ONLY the project files
    const loadedFiles = project.files || {};
    console.log('Loading files:', Object.keys(loadedFiles));
    
    // Force clear everything first
    setFiles({});
    setCurrentProject(null);
    setActiveFile('');
    setMdast(null);
    setFrontmatter({});
    setReferences({});
    setLastValidParseResult(null);
    
    // Then set the new stuff on next tick
    setTimeout(() => {
      setFiles(loadedFiles);
      setCurrentProject(project);
      onFilesChange(loadedFiles);
      
      // Set active file - make sure it exists
      const fileNames = Object.keys(loadedFiles);
      console.log('Available files:', fileNames);
      
      let targetFile = 'index.md';
      if (!loadedFiles['index.md'] && fileNames.length > 0) {
        targetFile = fileNames.find(f => f.endsWith('.md')) || fileNames[0];
      }
      
      console.log('Setting active file to:', targetFile);
      setActiveFile(targetFile);
    }, 50);
  };

  const handleNewProject = () => {
    const defaultFiles = {
      "index.md": `---
title: New MyST Project
subtitle: A fresh start with MyST Markdown
---

# Welcome to MyST

Start building your project here!
`,
      "myst.yml": `version: 1
project:
  title: My MyST Project
  
site:
  template: book-theme
`
    };
    
    setFiles(defaultFiles);
    onFilesChange(defaultFiles);
    setCurrentProject(null);
    setActiveFile('index.md');
  };

  // Get save button display based on status
  const getSaveButtonContent = () => {
    switch (saveStatus) {
      case 'saving':
        return '💾 Saving...';
      case 'saved':
        return '✅ Saved';
      case 'error':
        return '❌ Error';
      default:
        return '💾 Save';
    }
  };

  const getSaveButtonClass = () => {
    switch (saveStatus) {
      case 'saving':
        return 'bg-yellow-600 hover:bg-yellow-700';
      case 'saved':
        return 'bg-green-600 hover:bg-green-700';
      case 'error':
        return 'bg-red-600 hover:bg-red-700';
      default:
        return 'bg-blue-600 hover:bg-blue-700';
    }
  };

  return (
    <div className={classnames('flex h-full max-h-full overflow-hidden', className)} style={{ height: '75vh', minHeight: '400px', maxHeight: '1400px' }}>
      {/* File Explorer - always rendered, but collapsed or expanded */}
      <FileExplorer
        files={files}
        activeFile={activeFile}
        onFileSelect={setActiveFile}
        onFileAdd={handleFileAdd}
        onFileDelete={handleFileDelete}
        isCollapsed={isFileExplorerCollapsed}
        onToggleCollapse={() => setIsFileExplorerCollapsed(!isFileExplorerCollapsed)}
      />

      {/* Editor - always 50% of remaining space */}
      <div className="flex-1 flex flex-col overflow-hidden min-h-0">
        <div className="flex items-center justify-between px-3 py-2 bg-gray-800 border-b border-gray-700 flex-shrink-0">
          <div className="flex items-center gap-2">
            {/* Remove the toggle button from here since it's now in FileExplorer */}
            <span className="text-sm text-gray-300">
              {currentProject ? currentProject.name : 'MyST Sandbox'}
            </span>
            {currentProject && (
              <span className="text-xs text-blue-400 bg-blue-900 px-2 py-1 rounded">saved</span>
            )}
            <span className="text-xs text-gray-500">Ctrl+S to save</span>
          </div>
          
          <div className="flex gap-2">
            <button
              onClick={() => setShowProjectBrowser(true)}
              className="px-3 py-1 text-xs rounded bg-gray-600 text-white hover:bg-gray-700"
              title="Browse Projects"
            >
              📁 Projects
            </button>
            <button
              onClick={handleNewProject}
              className="px-3 py-1 text-xs rounded bg-purple-600 text-white hover:bg-purple-700"
              title="New Project"
            >
              + New
            </button>
            <button
              onClick={() => setShowSaveModal(true)}
              disabled={saveStatus === 'saving'}
              className={`px-3 py-1 text-xs rounded text-white ${getSaveButtonClass()} disabled:opacity-50 disabled:cursor-not-allowed`}
            >
              {getSaveButtonContent()}
            </button>
            {currentProject && (
              <a
                href={`/project/${currentProject.id}`}
                target="_blank"
                rel="noopener noreferrer"
                className="px-3 py-1 text-xs rounded bg-indigo-600 text-white hover:bg-indigo-700"
              >
                🌐 View Live
              </a>
            )}
            <button
              onClick={handleRender}
              disabled={isRendering}
              className="px-3 py-1 text-xs rounded bg-green-600 text-white hover:bg-green-700 disabled:opacity-50 disabled:cursor-not-allowed"
            >
              {isRendering ? 'Rendering...' : '🚀 Export Static'}
            </button>
          </div>
        </div>
        
        <CodeEditor
          filename={activeFile}
          content={activeFileContent}
          onChange={(content) => handleFileChange(activeFile, content)}
        />
      </div>

      {/* MyST Preview - always 50% of remaining space */}
      <div className="w-1/2 border-l border-gray-700 flex flex-col overflow-hidden min-h-0">
        <div className="px-3 py-2 bg-gray-800 border-b border-gray-700 flex-shrink-0">
          <span className="text-sm text-gray-300">
            MyST Preview {shouldShowPreview ? `- ${activeFile}` : ''}
          </span>
        </div>
        
        <div className="flex-1 overflow-auto p-4 myst min-h-0">
          {renderError && (
            <div className="mb-4 p-3 bg-red-100 border border-red-400 text-red-700 rounded">
              <strong>Render Error:</strong> {renderError}
            </div>
          )}
          
          {shouldShowPreview ? (
            mdast ? (
              <ArticleProvider references={references} frontmatter={frontmatter}>
                <GridSystemProvider>
                  <FrontmatterBlock frontmatter={frontmatter} />
                  <MyST ast={references.article || mdast} />
                </GridSystemProvider>
              </ArticleProvider>
            ) : (
              <div className="text-gray-400 text-center">
                {activeFileContent.trim() ? 'Parsing MyST...' : 'Start typing to see preview'}
              </div>
            )
          ) : (
            <div className="text-gray-500 text-center mt-16">
              <div className="text-6xl mb-4">📝</div>
              <div className="text-lg mb-2">Select a Markdown file to preview</div>
              <div className="text-sm opacity-75">Current: {activeFile}</div>
            </div>
          )}
        </div>
      </div>

      {/* Modals */}
      {showSaveModal && (
        <SaveProjectModal
          files={files}
          currentProject={currentProject}
          onSave={handleProjectSave}
          onClose={() => setShowSaveModal(false)}
        />
      )}

      {showProjectBrowser && (
        <ProjectBrowser
          currentProjectId={currentProject?.id}
          onLoadProject={handleProjectLoad}
          onClose={() => setShowProjectBrowser(false)}
        />
      )}

      {/* Render Success Popup */}
      {showRenderPopup && renderResult?.status === 'success' && (
        <RenderSuccessPopup
          siteUrl={renderResult.site_url}
          onClose={() => setShowRenderPopup(false)}
        />
      )}
    </div>
  );
}

Now all of this is super important but the main thing I want to harp on is this right here

  const parseMyst = (content: string) =>
    mystParse(content, {
      markdownit: { linkify: true },
      directives: [
        cardDirective,
        ...gridDirectives,
        ...tabDirectives,
        proofDirective,
        ...exerciseDirectives,
      ],
      roles: [buttonRole],
      vfile,
    });

  const mdast = parseMyst(md);
  
  const linkTransforms = [
    new WikiTransformer(),
    new GithubTransformer(),
    new DOITransformer(),
    new RRIDTransformer(),
    new RORTransformer(),
  ];

  // Frontmatter extraction (same as original)
  const firstParent = mdast.children[0]?.type === 'block' ? mdast.children[0] : mdast;
  const firstNode = firstParent.children?.[0];
  let frontmatterRaw = {};
  const firstIsYaml = firstNode?.type === 'code' && firstNode?.lang === 'yaml';
  if (firstIsYaml) {
    try {
      frontmatterRaw = (yamlLoad(firstNode.value) as Record<string, any>) || {};
      (firstNode as any).type = '__delete__';
    } catch (err) {
      fileError(vfile, 'Invalid YAML frontmatter', {
        note: (err as Error).message,
        ruleId: RuleId.frontmatterIsYaml,
      });
    }
  }
  
  const possibleNull = remove(mdast, '__delete__');
  if (possibleNull === null) {
    remove(mdast, { cascade: false }, '__delete__');
  }

  const validatedFrontmatter = validatePageFrontmatter(frontmatterRaw, {
    property: 'frontmatter',
    messages: {},
  });

  const references = {
    cite: { order: [], data: {} },
  };

  const state = new ReferenceState('', {
    frontmatter: validatedFrontmatter,
    vfile,
  });

  visit(mdast, (n) => {
    if (n.type === 'cite') {
      n.error = true;
    }
  });

  // Apply all transforms for full MyST functionality
  unified()
    .use(reconstructHtmlPlugin)
    .use(htmlPlugin)
    .use(basicTransformationsPlugin, { parser: parseMyst })
    .use(mathPlugin, { macros: validatedFrontmatter?.math ?? {} })
    .use(glossaryPlugin)
    .use(abbreviationPlugin, { abbreviations: validatedFrontmatter.abbreviations })
    .use(enumerateTargetsPlugin, { state })
    .use(linksPlugin, { transformers: linkTransforms })
    .use(footnotesPlugin)
    .use(joinGatesPlugin)
    .use(resolveReferencesPlugin, { state })
    .use(keysPlugin)
    .runSync(mdast as any, vfile);

  const html = mystToHtml(JSON.parse(JSON.stringify(mdast)));

  return { 
    mdast, 
    html, 
    frontmatter: validatedFrontmatter,
    references: { ...references, article: mdast }
  };
}

If you put in the time to do this then you'll get really good looking myst out the backend. Hope is that typst continues to upgrade their support for typst watch --format html for crossreferences in the future, but with myst we can do really cool things with publishing scientific content that it's just like why not? If people write in vanilla markdown they'll get vanilla markdown. Why not let me have all that functionality?

I know why! Because it's a lot of work. Well I've been busting my hump with claude to get up to speed on the state of rendering markdown in html over past few weeks and it's been an enriching experience.

Fuck the posts! I want the machines that are making 'em hahaha.