Waffle1412 / Typst Compiler Utility

#!/usr/bin/env bash # Default values jobs=0 # 0 means auto-detect output_dir="pdfs" verbose=0 quiet=0 create_dirs=0 dry_run=0 skip_newer=0 show_progress=1 force=0 select_mode=0 use_cache=1 # Enable caching by default cache_dir="${XDG_CACHE_HOME:-$HOME/.cache}/typst_compiler" dependency_scan=1 # Enable dependency scanning by default compile_log="typst_compile_errors.log" move_log="typst_move_errors.log" declare -a exclude_patterns # Show usage/help information show_help() { cat << EOF Usage: $(basename "$0") [OPTIONS] Compile Typst files and move generated PDFs to a designated directory. Options: -j, --jobs NUM Number of parallel jobs (default: auto-detect) -o, --output-dir DIR Output directory name (default: pdfs) -v, --verbose Increase verbosity -q, --quiet Suppress most output -c, --create-dirs Create output directories if they don't exist -d, --dry-run Show what would be done without doing it -s, --skip-newer Skip compilation if PDF is newer than source -S, --select Interactive selection mode using skim -f, --force Force compilation even if PDF exists --no-progress Disable progress bar --no-cache Disable compilation caching --clear-cache Clear the cache before compiling --no-deps Disable dependency scanning --cache-dir DIR Custom cache directory (default: ~/.cache/typst_compiler) --compile-log FILE Custom location for compilation log (default: $compile_log) --move-log FILE Custom location for move log (default: $move_log) -e, --exclude PATTERN Exclude files matching pattern (can be used multiple times) -h, --help Show this help message and exit Examples: $(basename "$0") -j 4 -o output -c $(basename "$0") --verbose --skip-newer $(basename "$0") -S # Select files to compile interactively $(basename "$0") -e "**/test/**" -e "**/draft/**" EOF } # Parse command line arguments while [[ $# -gt 0 ]]; do case $1 in -j|--jobs) jobs="$2" shift 2 ;; -o|--output-dir) output_dir="$2" shift 2 ;; -v|--verbose) verbose=1 shift ;; -q|--quiet) quiet=1 shift ;; -c|--create-dirs) create_dirs=1 shift ;; -d|--dry-run) dry_run=1 shift ;; -s|--skip-newer) skip_newer=1 shift ;; -S|--select) select_mode=1 shift ;; -f|--force) force=1 shift ;; --no-progress) show_progress=0 shift ;; --no-cache) use_cache=0 shift ;; --clear-cache) rm -rf "${cache_dir}" shift ;; --no-deps) dependency_scan=0 shift ;; --cache-dir) cache_dir="$2" shift 2 ;; --compile-log) compile_log="$2" shift 2 ;; --move-log) move_log="$2" shift 2 ;; -e|--exclude) exclude_patterns+=("$2") shift 2 ;; -h|--help) show_help exit 0 ;; *) echo "Unknown option: $1" show_help exit 1 ;; esac done # Check for conflicting options if [ "$verbose" -eq 1 ] && [ "$quiet" -eq 1 ]; then echo "Error: Cannot use both --verbose and --quiet" exit 1 fi # Check if required tools are installed check_tool() { if ! command -v "$1" &> /dev/null; then echo "$1 is not installed. Please install it first." echo "On most systems: $2" exit 1 fi } check_tool "fd" "cargo install fd-find or apt/brew install fd-find" check_tool "rg" "cargo install ripgrep or apt/brew install ripgrep" check_tool "parallel" "apt/brew install parallel" check_tool "sha256sum" "Built-in on most Linux systems, on macOS: brew install coreutils" # Check for skim and bat if select mode is enabled if [ "$select_mode" -eq 1 ]; then check_tool "sk" "cargo install skim or apt/brew install skim" check_tool "bat" "cargo install bat or apt/brew install bat" fi # ANSI color codes GREEN='\033[0;32m' BLUE='\033[0;34m' YELLOW='\033[0;33m' RED='\033[0;31m' CYAN='\033[0;36m' PURPLE='\033[0;35m' NC='\033[0m' # No Color BOLD='\033[1m' # Nerd font icons ICON_SUCCESS="󰄬 " ICON_ERROR="󰅚 " ICON_WORKING="󰄾 " ICON_COMPILE="󰈙 " ICON_MOVE="󰆐 " ICON_COMPLETE="󰄲 " ICON_SUMMARY="󰋽 " ICON_INFO="󰋼 " ICON_DEBUG="󰃤 " ICON_SELECT="󰓾 " ICON_CACHE="󰆏 " # Logging functions log_debug() { if [ "$verbose" -eq 1 ]; then echo -e "${BLUE}$ICON_DEBUG${NC} $*" fi } log_info() { if [ "$quiet" -eq 0 ]; then echo -e "${CYAN}$ICON_INFO${NC} $*" fi } log_warning() { if [ "$quiet" -eq 0 ]; then echo -e "${YELLOW}⚠️ ${NC} $*" fi } log_error() { echo -e "${RED}${BOLD}$ICON_ERROR${NC} $*" } log_success() { if [ "$quiet" -eq 0 ]; then echo -e "${GREEN}${BOLD}$ICON_SUCCESS${NC} $*" fi } # Create a directory for temporary files temp_dir=$(mktemp -d) compile_failures_dir="$temp_dir/compile_failures" move_failures_dir="$temp_dir/move_failures" progress_dir="$temp_dir/progress" mkdir -p "$compile_failures_dir" "$move_failures_dir" "$progress_dir" # Create cache directories if caching is enabled if [ "$use_cache" -eq 1 ]; then mkdir -p "${cache_dir}/hashes" "${cache_dir}/deps" "${cache_dir}/pdfs" log_debug "Using cache directory: $cache_dir" fi # Create a lock file for progress updates and a flag for final progress progress_lock="/tmp/typst_progress_lock" final_progress_file="$temp_dir/final_progress" touch "$final_progress_file" # Initialize log files > "$compile_log" > "$move_log" # Store current working directory CWD=$(pwd) log_info "Starting Typst compilation process..." # Build exclude arguments for fd fd_exclude_args=() for pattern in "${exclude_patterns[@]}"; do fd_exclude_args+=("-E" "$pattern") done # Create a list of files to process typst_files_list="$temp_dir/typst_files.txt" if [ "$select_mode" -eq 1 ]; then log_info "${PURPLE}$ICON_SELECT${NC} Interactive selection mode enabled" log_info "Use TAB to select multiple files, ENTER to confirm" # Set up bat configuration for Typst syntax highlighting bat_config="$temp_dir/bat_config" mkdir -p "$bat_config/syntaxes" export BAT_CONFIG_PATH="$bat_config" # Use skim with bat for preview to select files selected_files="$temp_dir/selected_files.txt" # Find all eligible .typ files for selection fd '\.typ$' --type f "${fd_exclude_args[@]}" . > "$typst_files_list.all" # Check if we found any files if [ ! -s "$typst_files_list.all" ]; then log_error "No .typ files found matching your criteria." rm -rf "$temp_dir" exit 0 fi # Prepare preview command for skim: use bat with custom syntax for .typ files preview_cmd="bat --color=always --style=numbers --map-syntax='*.typ:Markdown' {} 2>/dev/null || cat {}" # Use skim with bat preview to select files cat "$typst_files_list.all" | sk --multi \ --preview "$preview_cmd" \ --preview-window "right:70%" \ --height "80%" \ --prompt "Select .typ files to compile: " \ --header "TAB: Select multiple files, ENTER: Confirm, CTRL-C: Cancel" \ --no-mouse > "$selected_files" # Check if user selected any files if [ ! -s "$selected_files" ]; then log_error "No files selected. Exiting." rm -rf "$temp_dir" exit 0 fi # Use the selected files instead of all discovered files cp "$selected_files" "$typst_files_list" total_selected=$(wc -l < "$typst_files_list") total_available=$(wc -l < "$typst_files_list.all") log_info "Selected ${BOLD}$total_selected${NC} out of ${BOLD}$total_available${NC} available files" else # Normal mode - process all files fd '\.typ$' --type f "${fd_exclude_args[@]}" . > "$typst_files_list" log_info "Found ${BOLD}$(wc -l < "$typst_files_list")${NC} Typst files to process" fi total_files=$(wc -l < "$typst_files_list") # Function to extract Typst imports from a file extract_dependencies() { local file="$1" # Match both import formats: #import "file.typ" and #include "file.typ" # Also handle package imports: #import "@preview:0.1.0" # Return full paths to local imports, skipping package imports rg -U "#(import|include) \"(?!@)([^\"]+)\"" -r "$CWD/\$2" "$file" | \ while read -r dep; do # Resolve relative paths correctly if [[ "$dep" != /* ]]; then # If not absolute path, make it relative to the file's directory file_dir=$(dirname "$file") dep="$file_dir/$dep" fi # Normalize path dep=$(realpath --relative-to="$CWD" "$dep" 2>/dev/null || echo "$dep") # Only output if the dependency exists and is a .typ file if [[ "$dep" == *.typ ]] && [ -f "$dep" ]; then echo "$dep" fi done | sort -u # Sort and remove duplicates } # Function to build dependency graph for all files build_dependency_graph() { log_info "${ICON_CACHE} Building dependency graph..." # Create a deps file for each Typst file while read -r file; do file_hash=$(echo "$file" | md5sum | cut -d' ' -f1) deps_file="${cache_dir}/deps/${file_hash}.deps" # Skip if deps file is fresh and not forced if [ "$force" -eq 0 ] && [ -f "$deps_file" ] && [ "$file" -ot "$deps_file" ]; then log_debug "Using cached dependencies for $file" continue fi # Extract dependencies and save to deps file extract_dependencies "$file" > "$deps_file" log_debug "Updated dependencies for $file" done < "$typst_files_list" # Create sorted compilation order based on dependencies # Files with fewer dependencies (or no dependencies) come first if [ -f "$temp_dir/compile_order.txt" ]; then rm "$temp_dir/compile_order.txt" fi # First pass: count dependencies for each file declare -A dep_counts while read -r file; do file_hash=$(echo "$file" | md5sum | cut -d' ' -f1) deps_file="${cache_dir}/deps/${file_hash}.deps" # Count dependencies if [ -f "$deps_file" ]; then dep_count=$(wc -l < "$deps_file") else dep_count=0 fi dep_counts["$file"]=$dep_count done < "$typst_files_list" # Second pass: sort files by dependency count (ascending) while read -r file; do echo "${dep_counts["$file"]} $file" done < "$typst_files_list" | sort -n | cut -d' ' -f2- > "$temp_dir/compile_order.txt" log_debug "Created optimized compilation order" } # Function to check if a file has changed since last compilation has_file_changed() { local file="$1" local file_hash=$(echo "$file" | md5sum | cut -d' ' -f1) local hash_file="${cache_dir}/hashes/${file_hash}.sha256" # Force recompilation if requested if [ "$force" -eq 1 ]; then log_debug "Force recompilation of $file" return 0 # File considered changed fi # Check if hash file exists if [ ! -f "$hash_file" ]; then log_debug "No previous hash for $file" return 0 # File considered changed fi # Compare current file hash with stored hash local current_hash=$(sha256sum "$file" | cut -d' ' -f1) local stored_hash=$(cat "$hash_file") if [ "$current_hash" != "$stored_hash" ]; then log_debug "File $file has changed since last compilation" return 0 # File has changed fi # Check if any dependencies have changed if [ "$dependency_scan" -eq 1 ]; then local deps_file="${cache_dir}/deps/${file_hash}.deps" if [ -f "$deps_file" ]; then while read -r dep; do if has_file_changed "$dep"; then log_debug "Dependency $dep of $file has changed" return 0 # Dependency has changed fi done < "$deps_file" fi fi log_debug "File $file and its dependencies are unchanged" return 1 # File and dependencies haven't changed } # Function to update file hash after compilation update_file_hash() { local file="$1" local file_hash=$(echo "$file" | md5sum | cut -d' ' -f1) local hash_file="${cache_dir}/hashes/${file_hash}.sha256" # Update hash file sha256sum "$file" | cut -d' ' -f1 > "$hash_file" log_debug "Updated hash for $file" } # Function to cache compiled PDF cache_pdf() { local src_file="$1" local pdf_file="$2" if [ ! -f "$pdf_file" ]; then log_debug "No PDF file to cache: $pdf_file" return 1 fi local file_hash=$(echo "$src_file" | md5sum | cut -d' ' -f1) local cached_pdf="${cache_dir}/pdfs/${file_hash}.pdf" # Copy PDF to cache cp "$pdf_file" "$cached_pdf" log_debug "Cached PDF for $src_file" return 0 } # Function to retrieve cached PDF get_cached_pdf() { local src_file="$1" local target_file="$2" local file_hash=$(echo "$src_file" | md5sum | cut -d' ' -f1) local cached_pdf="${cache_dir}/pdfs/${file_hash}.pdf" if [ ! -f "$cached_pdf" ]; then log_debug "No cached PDF for $src_file" return 1 fi # Copy cached PDF to target location cp "$cached_pdf" "$target_file" log_debug "Retrieved cached PDF for $src_file" return 0 } # Function to update progress bar during processing update_progress_during() { # If progress is disabled, do nothing if [ "$show_progress" -eq 0 ]; then return 0 fi ( # Try to acquire lock, but don't wait if busy flock -n 200 || return 0 completed=$(find "$progress_dir" -type f | wc -l) percent=$((completed * 100 / total_files)) bar_length=50 filled_length=$((bar_length * completed / total_files)) # Create the progress bar bar="" for ((i=0; i<bar_length; i++)); do if [ $i -lt $filled_length ]; then bar="${bar}█" else bar="${bar}░" fi done # Calculate success and failure counts success=$((completed - $(find "$compile_failures_dir" "$move_failures_dir" -type f | wc -l))) compile_fails=$(find "$compile_failures_dir" -type f | wc -l) move_fails=$(find "$move_failures_dir" -type f | wc -l) # Clear the previous line and print the updated progress echo -ne "\r\033[K" echo -ne "${PURPLE}$ICON_WORKING Progress: ${GREEN}$bar ${BOLD}${percent}%${NC} " echo -ne "[${GREEN}${success}✓${NC}|${RED}${compile_fails}✗${NC}|${YELLOW}${move_fails}!${NC}] " echo -ne "${BLUE}($completed/$total_files)${NC}" # If all files are processed, print the completion message ONLY ONCE if [ $completed -eq $total_files ] && [ ! -e "$final_progress_file.done" ]; then touch "$final_progress_file.done" echo -e "\n${GREEN}${BOLD}$ICON_COMPLETE All files processed!${NC}" fi ) 200>"$progress_lock" } # Function to show final progress (called only once at the end) show_final_progress() { # If progress is disabled, do nothing if [ "$show_progress" -eq 0 ]; then return 0 fi ( flock -w 1 200 # Only proceed if the final progress hasn't been shown yet if [ -e "$final_progress_file.done" ]; then return 0 fi completed=$(find "$progress_dir" -type f | wc -l) percent=$((completed * 100 / total_files)) bar_length=50 filled_length=$((bar_length * completed / total_files)) # Create the progress bar bar="" for ((i=0; i<bar_length; i++)); do if [ $i -lt $filled_length ]; then bar="${bar}█" else bar="${bar}░" fi done # Calculate success and failure counts success=$((completed - $(find "$compile_failures_dir" "$move_failures_dir" -type f | wc -l))) compile_fails=$(find "$compile_failures_dir" -type f | wc -l) move_fails=$(find "$move_failures_dir" -type f | wc -l) # Clear the previous line and print the updated progress echo -ne "\r\033[K" echo -ne "${PURPLE}$ICON_WORKING Progress: ${GREEN}$bar ${BOLD}${percent}%${NC} " echo -ne "[${GREEN}${success}✓${NC}|${RED}${compile_fails}✗${NC}|${YELLOW}${move_fails}!${NC}] " echo -ne "${BLUE}($completed/$total_files)${NC}" # Mark final progress as shown touch "$final_progress_file.done" # If all files are processed, print the completion message if [ $completed -eq $total_files ]; then echo -e "\n${GREEN}${BOLD}$ICON_COMPLETE All files processed!${NC}" fi ) 200>"$progress_lock" } # Function to process a single .typ file process_file() { typfile="$1" file_id=$(echo "$typfile" | md5sum | cut -d' ' -f1) # Get the directory containing the .typ file typdir=$(dirname "$typfile") # Get the filename without path filename=$(basename "$typfile") # Get the filename without extension basename="${filename%.typ}" # Check if output directory exists or should be created target_dir="$typdir/$output_dir" if [ ! -d "$target_dir" ]; then if [ "$create_dirs" -eq 1 ]; then if [ "$dry_run" -eq 0 ]; then mkdir -p "$target_dir" log_debug "Created directory: $target_dir" else log_debug "[DRY RUN] Would create directory: $target_dir" fi else # Skip this file if output directory doesn't exist and --create-dirs not specified log_debug "Skipping $typfile (no $output_dir directory)" touch "$progress_dir/$file_id" update_progress_during return 0 fi fi target_pdf="$target_dir/$basename.pdf" # Skip if PDF is newer than source and --skip-newer is specified if [ "$skip_newer" -eq 1 ] && [ -f "$target_pdf" ]; then if [ "$typfile" -ot "$target_pdf" ] && [ "$force" -eq 0 ]; then log_debug "Skipping $typfile (PDF is newer)" touch "$progress_dir/$file_id" update_progress_during return 0 fi fi # Check if file has changed (if caching is enabled) if [ "$use_cache" -eq 1 ] && [ "$dry_run" -eq 0 ]; then if ! has_file_changed "$typfile"; then # Try to retrieve cached PDF if get_cached_pdf "$typfile" "$target_pdf"; then log_debug "Using cached PDF for $typfile" touch "$progress_dir/$file_id" update_progress_during return 0 fi fi fi # Create a temporary file for capturing compiler output temp_output="$temp_dir/output_${file_id}.log" # Add a header to the log before compilation { echo -e "\n===== COMPILING: $typfile =====" echo "$(date)" } > "$temp_output.header" # In dry run mode, just log what would be done if [ "$dry_run" -eq 1 ]; then log_debug "[DRY RUN] Would compile: $typfile" log_debug "[DRY RUN] Would move to: $target_pdf" touch "$progress_dir/$file_id" update_progress_during return 0 fi # Compile the .typ file using typst with --root flag and capture all output if ! typst compile --root "$CWD" "$typfile" > "$temp_output.stdout" 2> "$temp_output.stderr"; then # Store the failure echo "$typfile" > "$compile_failures_dir/$file_id" log_debug "Compilation failed for $typfile" # Combine stdout and stderr cat "$temp_output.stdout" "$temp_output.stderr" > "$temp_output.combined" # Filter the output to only include error messages using ripgrep rg "error:" -A 20 "$temp_output.combined" > "$temp_output.errors" || true # Lock the log file to avoid concurrent writes corrupting it ( flock -w 1 201 cat "$temp_output.header" "$temp_output.errors" >> "$compile_log" echo -e "\n" >> "$compile_log" ) 201>"$compile_log.lock" else # Check if the output PDF exists temp_pdf="$typdir/$basename.pdf" if [ -f "$temp_pdf" ]; then # Cache the PDF if caching is enabled if [ "$use_cache" -eq 1 ]; then cache_pdf "$typfile" "$temp_pdf" # Update file hash update_file_hash "$typfile" fi # Try to move the output PDF to the output directory move_header="$temp_dir/move_${file_id}.header" { echo -e "\n===== MOVING: $typfile =====" echo "$(date)" } > "$move_header" if ! mv "$temp_pdf" "$target_dir/" 2> "$temp_output.move_err"; then echo "$typfile -> $target_pdf" > "$move_failures_dir/$file_id" log_debug "Failed to move $basename.pdf to $target_dir/" # Lock the log file to avoid concurrent writes corrupting it ( flock -w 1 202 cat "$move_header" "$temp_output.move_err" >> "$move_log" echo "Failed to move $temp_pdf to $target_dir/" >> "$move_log" ) 202>"$move_log.lock" else log_debug "Moved $basename.pdf to $target_dir/" fi else # This is a fallback check in case typst doesn't return error code echo "$typfile" > "$compile_failures_dir/$file_id" log_debug "Compilation completed without errors but no PDF was generated for $typfile" # Lock the log file to avoid concurrent writes corrupting it ( flock -w 1 201 echo "Compilation completed without errors but no PDF was generated" >> "$compile_log" ) 201>"$compile_log.lock" fi fi # Mark this file as processed (for progress tracking) touch "$progress_dir/$file_id" # Update the progress bar update_progress_during } export -f process_file export -f update_progress_during export -f show_final_progress export -f log_debug export -f log_info export -f log_warning export -f log_error export -f log_success export -f has_file_changed export -f update_file_hash export -f cache_pdf export -f get_cached_pdf export CWD export temp_dir export compile_failures_dir export move_failures_dir export progress_dir export final_progress_file export progress_lock export compile_log export move_log export cache_dir export total_files export GREEN BLUE YELLOW RED CYAN PURPLE NC BOLD export ICON_SUCCESS ICON_ERROR ICON_WORKING ICON_COMPILE ICON_MOVE ICON_COMPLETE ICON_SUMMARY ICON_INFO ICON_DEBUG ICON_CACHE export verbose export quiet export output_dir export create_dirs export dry_run export skip_newer export show_progress export force export use_cache export dependency_scan # Determine the number of CPU cores and use that many parallel jobs (if not specified) if [ "$jobs" -eq 0 ]; then jobs=$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 4) fi log_info "Using ${BOLD}$jobs${NC} parallel jobs for compilation" # Build dependency graph if dependency scanning is enabled if [ "$dependency_scan" -eq 1 ] && [ "$total_files" -gt 0 ]; then build_dependency_graph # Use optimized compilation order cp "$temp_dir/compile_order.txt" "$typst_files_list" log_info "Optimized compilation order based on dependencies" fi # Initialize progress bar if showing progress if [ "$show_progress" -eq 1 ]; then update_progress_during fi # Process files in parallel with --will-cite to suppress citation notice if [ "$total_files" -gt 0 ]; then cat "$typst_files_list" | parallel --will-cite --jobs "$jobs" process_file # Wait a moment for any remaining progress updates to complete sleep 0.5 # Show the final progress exactly once if showing progress if [ "$show_progress" -eq 1 ]; then show_final_progress fi # Print summary of failures if [ "$quiet" -eq 0 ]; then echo -e "\n${BOLD}${PURPLE}$ICON_SUMMARY Processing Summary${NC}" fi # Collect all failure files compile_failures=$(find "$compile_failures_dir" -type f | wc -l) move_failures=$(find "$move_failures_dir" -type f | wc -l) if [ "$compile_failures" -eq 0 ] && [ "$move_failures" -eq 0 ]; then log_success "All files processed successfully." else if [ "$compile_failures" -gt 0 ]; then echo -e "\n${RED}${BOLD}$ICON_ERROR Compilation failures:${NC}" find "$compile_failures_dir" -type f -exec cat {} \; | sort | while read -r failure; do echo -e "${RED}- $failure${NC}" done echo -e "${BLUE}See $compile_log for detailed error messages.${NC}" fi if [ "$move_failures" -gt 0 ]; then echo -e "\n${YELLOW}${BOLD}$ICON_ERROR Move failures:${NC}" find "$move_failures_dir" -type f -exec cat {} \; | sort | while read -r failure; do echo -e "${YELLOW}- $failure${NC}" done echo -e "${BLUE}See $move_log for detailed error messages.${NC}" fi fi # Cache usage statistics (if enabled and not in quiet mode) if [ "$use_cache" -eq 1 ] && [ "$quiet" -eq 0 ]; then cache_files=$(find "${cache_dir}/pdfs" -type f | wc -l) cache_size=$(du -sh "${cache_dir}" 2>/dev/null | cut -f1) echo -e "\n${CYAN}${ICON_CACHE} Cache statistics:${NC}" echo -e " - Cached files: ${BOLD}$cache_files${NC}" echo -e " - Cache size: ${BOLD}$cache_size${NC}" echo -e " - Cache location: ${BOLD}$cache_dir${NC}" fi else log_warning "No .typ files found to process." fi # Clean up temporary directory and lock files rm -rf "$temp_dir" rm -f "$progress_lock" "$compile_log.lock" "$move_log.lock" log_success "Processing complete."