refactor(scripts)!: Redo split chapter file loading

BREAKING CHANGE: Books with split source files are now compiled
automatically without the loadchapters macro shenanigans. This requires
the sources to be sortable and any interleaved content needs to be in
sequential files.

rules/rules.mk

@@ -530,12 +530,17 @@ SILEFLAGS += $(call use_luas,$(LUAINCLUDES))
 
 preprocess_macros = $(CASILEDIR)/casile.m4 $(M4MACROS) $(PROJECTMACRO) $(TARGETMACROS_$1)
 
-$(BUILDDIR)/%-$(_processed).md: %.md $$(wildcard $(PROJECT)*.md $$*-$(_chapters)/*.md) $$(call preprocess_macros,$$*) | $(BUILDDIR) figures
+$(BUILDDIR)/%-$(_processed).md: %.md $$(shell $(_ENV) list_related_files.zsh mds $$*) $$(call preprocess_macros,$$*) | $(BUILDDIR) figures
 	if $(HIGHLIGHT_DIFF) && $(if $(PARENT),true,false); then
 		export FILTERS="$(filter %.m4,$^)"
 		branch2criticmark.zsh $(PARENT) $<
-	else
+	elif $(RG) -qw loadchapters $<; then
+		# For books using the old M4 macros to load chapters, only process the first dependency
 		$(M4) $(filter %.m4,$^) $<
+	else
+		# For books using the new related files finder, load all md files except the first dependency
+		# Add blank lines between files so headers aren't eaten, c.f. https://unix.stackexchange.com/a/628651/1925
+		$(M4) $(filter %.m4,$^) =($(SED) -s -e '$${p;g;}' $(filter %.md,$(wordlist 2,9999,$^)))
 	fi |
 		renumber_footnotes.pl |
 		$(and $(HEAD),head -n$(HEAD) |) \

scripts/import.zsh.in

@@ -41,4 +41,4 @@ import_${format} $bookid $input
 untrack $input
 commit "Import $format source as $bookid"
 
-"$CASILEDIR/scripts/split_chapters.zsh" 2 $bookid
+"$CASILEDIR/scripts/split_chapters.zsh" 3 $bookid

scripts/list_related_files.zsh.in

@@ -2,24 +2,34 @@
 set -e
 
 cmd=$1
-base=${2/.md}
+bookid=${2/.md}
 
-value() {
-  echo $@
-  exit 0
+function value () {
+    echo $@
+    exit 0
 }
 
 case $cmd in
-  meta)
-    value $base.yml
-    ;;
+    meta)
+        value $bookid.yml
+        ;;
+    mds)
+        alias git="@GIT@"
+        # Separate commands to set our own sort priority
+        git ls-files "$PROJECT.md" ||:
+        git ls-files "$PROJECT-*.md" ||:
+        git ls-files "$bookid.md" ||:
+        git ls-files "$bookid-**.md" ||:
+        exit 0
+        ;;
   srcid)
-    $0 meta $2 | read meta
-    if [[ -f $meta ]] && yq -e 'has("source")' $meta >/dev/null; then
-      yq -r '[.source[] | select(.type == "bookid")][0].text' $meta | read bookid
-      value $bookid
-    fi
-    ;;
+      alias yq="@YQ@"
+      $0 meta $2 | read meta
+      if [[ -f $meta ]] && yq -e 'has("source")' $meta >/dev/null; then
+          yq -r '[.source[] | select(.type == "bookid")][0].text' $meta | read bookid
+          value $bookid
+      fi
+      ;;
 esac
 
 exit 1

scripts/normalize_files.zsh.in

@@ -19,18 +19,14 @@ pandoc_args=(
 # TODO: do we need turkish filename hacks back?
 # $(SED) -n '/^#/{s/ı/i/g;p}' |
 
-# TODO: finish and use related files finder?
-# $CASILEDIR/scripts/list_related_files.zsh $bookid |
-
-git ls-files "$bookid-*/*.md" |
+$CASILEDIR/scripts/list_related_files.zsh mds $bookid |
 	xargs rg --files-without-match loadchapters.zsh |
 	while read file; do
 		basename $file | IFS='[-.]' read no _
 		dirname $file | read dir
 		rg -m1 '^#' $file | pandoc ${pandoc_args[@]} | read identifier
+		[[ $identifier == nil ]] && continue
 		git mv -k "$file" "$dir/$no-$identifier.md"
 	done
 
-echo done
-
 commit "Normalize $bookid filenames based on heading ids"

scripts/split_chapters.zsh.in

@@ -4,7 +4,6 @@ set -e
 source "$CASILEDIR/scripts/functions.zsh"
 
 alias pandoc="@PANDOC@"
-alias sponge="@SPONGE@"
 alias sed="@SED@"
 alias mkdir="@MKDIR_P@"
 
@@ -14,7 +13,7 @@ test -n $splitlevels
 bookid=$2
 test -n $bookid
 
-# TODO: actually support --top-level=part where parts are another level of splits
+# TODO: restore support for top level being chapters instead of parts
 part_no=0
 chapter_no=0
 section_no=0
@@ -34,27 +33,29 @@ while read -r line; do
 	[[ $line =~ "^\[\^1\]: .*" ]] && break
 
 	# Check for headers
-	if [[ $line =~ '^\\part\{.*\}$' ]]; then
-		of=$bookid.md
-	elif [[ $line =~ "^#+ .*" ]]; then
-		if [[ $line =~ "^# " ]]; then
+	if [[ $line =~ "^#+ .*" ]] || [[ $line =~ '^\\part\{.*\}$' ]]; then
+		if [[ $line =~ '^\\part\{.*\}$' ]] && [[ $splitlevels -ge 1 ]]; then
+			chapter_no=0
+			section_no=0
+			let part_no=$part_no+1
+			dir=${bookid}-parts
+			mkdir $dir
+			of=$dir/$(printf %03d $part_no).md
+		elif [[ $line =~ "^# " ]] && [[ $splitlevels -ge 2 ]]; then
 			section_no=0
 			[[ $line =~ ".*unnumbered.*" ]] || let chapter_no=$chapter_no+1
-			dir=${bookid}-chapters
+			dir=${bookid}-parts/$(printf %03d $part_no)-chapters
 			mkdir $dir
 			of=$dir/$(printf %03d $chapter_no).md
-			files+=($of)
-			>> $bookid.md <<< "esyscmd([[loadchapters.zsh \"$of\"]])"$'\n'
-		elif [[ $line =~ "^## " ]] && [[ $splitlevels -le 2 ]] then
+		elif [[ $line =~ "^## " ]] && [[ $splitlevels -ge 3 ]]; then
 			[[ $line =~ ".*unnumbered.*" ]] || let section_no=$section_no+1
-			dir=${bookid}-chapters/$(printf %03d $chapter_no)-sections/
+			dir=${bookid}-parts/$(printf %03d $part_no)-chapters/$(printf %03d $chapter_no)-sections
 			mkdir $dir
 			of=$dir/$(printf %03d $section_no).md
-			files+=($of)
-			>> $bookid.md <<< "esyscmd([[loadchapters.zsh \"$of\"]])"$'\n'
 		elif [[ $splitlevels -ge 3 ]] then
 			flunk 'Unsupported 3 level split'
 		fi
+		files+=($of)
 	fi
 
 	>> $of <<< "$line"

upgrade-markdown.sed

@@ -1,3 +1,4 @@
 s#zsh -c 'for f in *#loadchapters.zsh "#
 s#; do cat \$f; echo; done'#"#
 s#\\lang\([a-z]\+\){\([^}]\+\)}#[\2]{lang="\1"}#g
+/loadchapters.zsh /d