trimbib

#!/bin/bash
# Usage: trimbib [-l|--latex] [-t|--tex] [NAME_OF_BIBTEX_FILE] [-o|--output NAME_OF_OUTPUT_FILE]
# This script trims bib files using biber in the following way.
# 1. Only one of the duplicated entries is kept. (But which one? I am not sure what is biber's rule.)
# 2. Items are ordered by author, year, and the entry key.
# 3. Macros are sorted alphabetically.
# 4. The file is indented.
#
# If -l|--latex is present, then a BibLaTeX file will be produced;
# If -t|--tex is present, then a BibTeX file will be produced;
# if neither -l|--latex nor -t|--tex is present, then trimbib will try to decide the type of the
# input file and generate a file of the same type; if the type cannot the decided, then a BibTeX
# file will be generated.

USER_SPECIFY_LATEX=0
USER_SPECIFY_TEX=0
BIBFILE=""
OUTPUTFILE=""
TIME=$(date +%Y.%m.%d-%H.%M.%S)

# Log files
TRIMBIBLOG="trimbib.log"
CHECKBIBLOG="checkbib.log"

# Directory to contain the backups.
BAKDIR=./backup/

# Check whether biber is available with a high enough version.
BIBER_MINVER=2.17
if ! command -v biber &> /dev/null ; then
    printf "\nbiber version >= %s is needed, but biber is not found.\n\n"  "$BIBER_MINVER"
    exit 3
fi
BIBER_VER="$(biber --version | grep -Eo '[0-9]\.[0-9]+' | head -1)"
if (( $(echo "$BIBER_MINVER > $BIBER_VER" | bc -l) )); then
    printf "\nbiber version >= %s is needed, but the current version is %s.\n\n"  "$BIBER_MINVER" "$BIBER_VER"
    exit 3
fi

if [[ $# -gt 4 ]]; then
    printf "Usage: trimbib [-l|--latex] [-t|--tex] [BIBFILE_NAME] [-o|--output NAME_OF_OUTPUT_FILE]\n"
    exit 1
fi

# Parse the arguments
while [[ -n "$1" ]]; do
	case "$1" in
	-l|--latex)
        USER_SPECIFY_LATEX=1
		;;
    -t|--tex)
        USER_SPECIFY_TEX=1
        ;;
    -o|--output)
        if [[ $# -gt 1 ]]; then
            OUTPUTFILE="$2"
            shift
        fi
        ;;
	*)
        if [[ -z "$BIBFILE" ]]; then
            BIBFILE="$1"
        fi
        ;;
	esac
	shift
done

# Get the name of the bib file with full path.
# If $BIBFILE is a symlink, we work on its target.
# If the input is empty, then use the latest bib file under the current directory.
if [[ -z "$BIBFILE" ]]; then
    BIBFILE=$(find ./ -maxdepth 1 -type f -name "*.bib" -print0 | xargs -r -0 ls -1 -t | head -1)
fi
BIBFILE="$(readlink -f "$BIBFILE")"
if [[ -z "${BIBFILE##*/}" ]]; then
	printf "\nNo bib file found.\n"
	exit 2
elif [[ ! -f "$BIBFILE" ]]; then
    printf "\n%s does not exist.\n" "$BIBFILE"
	exit 2
fi

if grep -q "journaltitle[[:space:]]*=[[:space:]]*\|date[[:space:]]*=[[:space:]]*\|location[[:space:]]*=[[:space:]]*\|type[[:space:]]*=[[:space:]]*" "$BIBFILE" ; then
    USER_INPUT_LATEX=1
else
    USER_INPUT_LATEX=0
fi

if [[ $USER_SPECIFY_LATEX -eq 1 && $USER_SPECIFY_TEX -eq 1 ]] ; then
    printf "\n-l|--latex and -t|--t cannot be present simultaneously.\n"
    exit 1
elif [[ $USER_SPECIFY_LATEX -eq 1 ]] ; then
    BIBLATEX=1
elif [[ $USER_SPECIFY_LATEX -ne 1 ]] ; then
    BIBLATEX=0
elif [[ $USER_INPUT_LATEX -eq 1 ]] ; then
    BIBLATEX=1
else
    BIBLATEX=0
fi

# If no output file is specified, set OUTPUTFILE to $BIBFILE.
if [[ -z "$OUTPUTFILE" ]]; then
    OUTPUTFILE=$BIBFILE
fi
# Get the full path of $OUTPUTFILE. Note that we do not resolve the link if $OUTPUTFILE exists
# and is a symlink.
OUTPUTFILE="$(cd "$(dirname -- "$OUTPUTFILE")" && pwd)/$(basename -- "$OUTPUTFILE")"

# Check the bib file before trimming.
if [[ $USER_INPUT_LATEX -eq 1 ]] ; then
    checkbib -l "$BIBFILE"
else
    checkbib "$BIBFILE"
fi

read -n1 -s -r -p \
    $'Continue to trim the bib file? [Y/n] \nRevise it before trimming if it contains major mistakes.\n' KEY

if ! [[ "$KEY" == 'Y' || "$KEY" == 'y' || "$KEY" == "" ]]; then
    exit 1
fi

echo "**************** checkbib.log before trimming ****************" | cat - $CHECKBIBLOG > "tmp-$TIME" && mv "tmp-$TIME" $CHECKBIBLOG
mv $CHECKBIBLOG $CHECKBIBLOG.bak

# Temporary files.
INPUTBIB="inputbib-$TIME.bib"
OUTPUTBIB="outputbib-$TIME.bib"
TMPBIB="tmpbib-$TIME.bib"
BIBMACROS="bibmacros-$TIME"
BIBEROUT="biberout-$TIME"
BIBENTRIES="bibentries-$TIME"

cp "$BIBFILE" "$INPUTBIB"

# Pre-process the bib file if the output is a BibTeX file. This is necessary if the input is BibLaTeX.
LINEBREAKER="LINEBREAKERSTARTING$(date +%s)LINEBREAKERENDING"
if [[ $BIBLATEX -eq 0 ]] ; then
    sed '/^[[:space:]]*%/d' "$INPUTBIB" | sed 's/%.*$//' | awk -v ORS="$LINEBREAKER" 1 \
        | sed "s/$LINEBREAKER[[:space:]]*@/\n@/g" | sed "s/$LINEBREAKER//g" > "$TMPBIB"
    {
        grep -v "techreport\|phdthesis\|masterthesis" "$TMPBIB"
        grep techreport "$TMPBIB" | sed 's/^[[:space:]]*@report/@techreport/' | sed 's/type[[:space:]]*=[[:space:]]*{.*}[[:space:]]*,//'
        grep phdthesis "$TMPBIB" | sed 's/^[[:space:]]*@thesis/@phdthesis/' | sed 's/type[[:space:]]*=[[:space:]]*{.*}[[:space:]]*,//' \
            | sed 's/institution/school/'
        grep masterthesis "$TMPBIB" | sed 's/^[[:space:]]*@thesis/@masterthesis/' | sed 's/type[[:space:]]*=[[:space:]]*{.*}[[:space:]]*,//' \
            | sed 's/institution/school/'
    } > "$OUTPUTBIB"
    rm -f "$TMPBIB"
    sed -i 's/,[[:space:]]*journaltitle[[:space:]]*=[[:space:]]*/,journal=/' "$OUTPUTBIB"
    sed -i 's/,[[:space:]]*date[[:space:]]*=[[:space:]]*/,year=/' "$OUTPUTBIB"
    sed -i 's/,[[:space:]]*location[[:space:]]*=[[:space:]]*/,address=/' "$OUTPUTBIB"
    mv "$OUTPUTBIB" "$INPUTBIB"
fi

# Print the macros in $INPUTBIB to $BIBMACROS
printf "\nSorting the macros ..."
grep "^[[:space:]]*@string{\|^[[:space:]]*@String{\|^[[:space:]]*@STRING{" "$INPUTBIB" | sed 's/.*@/@/' \
    | sed 's/@String/@string/' | sed 's/@STRING/@string/' \
    | sed 's/\.[[:space:]]*"[[:space:]]*}/\."}/' \
    | sed 's/[[:space:]]*=[[:space:]]*/\ =\ /' \
    > "$BIBMACROS"


# Trim the bib entries by biber
printf "\nInvoking biber to trim the bib entries ...\n"

# Configuration of biber.
################################################################################
cat > biber.conf <<EOF
<?xml version="1.0"?>
<config>
	<output_align>true</output_align>
	<output_fieldcase>lower</output_fieldcase>

	<!--!!!The following lines make biber to sort the bib entries according to year, author, and title.!!!-->
	<sortingtemplate name="tool">
		<sort order="1" sort_direction="ascending">
			<sortitem order="1">author</sortitem>
		</sort>
		<sort order="2" sort_direction="ascending">
			<sortitem order="1">year</sortitem>
		</sort>
		<sort order="3" sort_direction="ascending">
			<sortitem order="1">entrykey</sortitem>
		</sort>
	</sortingtemplate>

	<!--!!!By default, biber removes the "nonstandard fields".!!!-->
	<!--!!!The following lines ensure that the nonstandard fields are retained in certain entries.!!!-->
    <!--!!!See https://tex.stackexchange.com/questions/415028/prevent-biber-tool-from-removing-non-standard-fields-in-bib-files !!!-->
  	<datamodel>
		<fields>
            <!--!!!Put the name of the desired fields here.!!!-->
	  		<field fieldtype="field" datatype="literal">_number</field>
	  		<field fieldtype="field" datatype="literal">_publisher</field>
	  		<field fieldtype="field" datatype="literal">note</field>
	  		<field fieldtype="field" datatype="literal">journal</field>
	  		<field fieldtype="field" datatype="literal">school</field>
	  		<field fieldtype="field" datatype="literal">address</field>
		</fields>
		<entryfields>
            <!--!!!For article, retain "_number", "_publisher", "note", "journal".!!!-->
	  		<entrytype>article</entrytype>
	  		<field>_number</field>
	  		<field>_publisher</field>
	  		<field>note</field>
	  		<field>journal</field>
		</entryfields>
		<entryfields>
            <!--!!!For masterthesis, retain "school", "address", "note".!!!-->
	  		<entrytype>masterthesis</entrytype>
	  		<field>school</field>
	  		<field>address</field>
	  		<field>note</field>
		</entryfields>
		<entryfields>
            <!--!!!For phdthesis, retain "school", "address", "note".!!!-->
	  		<entrytype>phdthesis</entrytype>
	  		<field>school</field>
	  		<field>address</field>
	  		<field>note</field>
		</entryfields>
		<entryfields>
            <!--!!!For techreport, retain "number", "institution", "address", "note".!!!-->
	  		<entrytype>techreport</entrytype>
	  		<field>number</field>
	  		<field>institution</field>
	  		<field>address</field>
	  		<field>note</field>
		</entryfields>
  	</datamodel>
EOF

if [[ $BIBLATEX -eq 1 ]] ; then
cat >> biber.conf <<EOF
</config>
EOF
else
cat >> biber.conf <<EOF
	<!--!!!By default, biber maps "masterthesis" to "thesis" with a "type = {masterthesis}".!!!-->
	<!--!!!It does similar things on "phdthesis".!!!-->
    <!--!!!In addition, it maps "techreport" to "report" with type={techreport}.!!!-->
	<!--!!!The following lines revokes this behavior.!!!-->
    <sourcemap>
        <maps datatype="bibtex">
            <map>
                <map_step map_type_source="mastersthesis" map_type_target="masterthesis" map_final="1"/>
            </map>
            <map>
                <map_step map_type_source="phdthesis" map_type_target="phdthesis" map_final="1"/>
            </map>
            <map>
                <map_step map_type_source="techreport" map_type_target="techreport" map_final="1"/>
            </map>
        </maps>
    </sourcemap>
</config>
EOF
fi
################################################################################

# Invoke biber.
# 1. "--output-safechars --output-safecharsset=full" tells biber to encode all non-ascii characters
#    into LaTeX macros. By default, this is disabled. Similar behavior can be invoked by
#    "--decodecharsset=null"; however, as of 2021-03-22, this approach seems buggy, because it will
#    cause biber to remove all the curly braces enclosing only one letter, e.g., {H}ilbert.
# 2. By default, biber uses "location" instead of "address", "journaltitle" instead of "journal",
#    and "date" instead of year (due to the difference between BibTeX and BibLaTeX). We use
#    "--output-field-replace" to disable such behavior.
if [[ $BIBLATEX -eq 1 ]] ; then
    biber --output-file "$BIBEROUT" \
        --tool --configfile=biber.conf \
        --output-safechars --output-safecharsset=full \
        --output_indent=4 "$INPUTBIB"
else
    biber --output-file "$BIBEROUT" \
        --tool --configfile=biber.conf \
        --output-safechars --output-safecharsset=full \
        --output_indent=4 \
        --output-field-replace=location:address,journaltitle:journal,date:year "$INPUTBIB"
fi

rm biber.conf
rm "$INPUTBIB"
rm "$INPUTBIB.blg"


# Print the trimmed bib entries into $BIBENTRIES.
touch "$BIBEROUT"  # $BIBEROUT is not generated if biber encounters errors.
gawk -v line="$(gawk '/@STRING|@String|@string/{n=NR}END{print n}' "$BIBEROUT")" 'NR>line' "$BIBEROUT" >  "$BIBENTRIES"

rm "$BIBEROUT"
if ! grep -q '[^[:space:]]' "$BIBENTRIES" ; then
    printf "\n%s is not a bib file or contains no bib entry. Abort.\n" "$BIBFILE"
    rm "$OUTPUTBIB" "$BIBMACROS" "$BIBENTRIES" "$CHECKBIBLOG.bak"
    exit 3
fi


# Generate the new bib file.
{
    printf "%% bib file generated at %s.\n" "$TIME"
    printf "%%\n"
    printf "%% The recommended format of the bib entry label is:\n"
    printf "%%\n"
    printf "%%     <AuthorLastnames_YearLetter>\n"
    printf "%%\n"
    printf "%%  For example, the key for an article by Yuan may be:\n"
    printf "%%\n"
    printf "%%     Yuan_2000a\n%%\n"
    printf "%% It is advised to use the abbreviations of names of serials in Math. Rev.\n"
    printf "%% See https://mathscinet.ams.org/msnhtml/serials.pdf .\n"
    printf "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n\n\n"
} > "$OUTPUTBIB"

#grep -q -v '^[[:space:]]*$' "$BIBMACROS"  > /dev/null  # Check whether $BIBMACROS is empty.
if grep -q -v '^[[:space:]]*$' "$BIBMACROS"  > /dev/null ; then
{
    printf "%% String macros for abbreviations of names of serials.\n"
    printf "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%MACRO_BEGIN%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n"
    sed -r '/^[[:space:]]*$/d' "$BIBMACROS" | sort -u
    printf "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%MACRO_END%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n\n"
} >> "$OUTPUTBIB"
fi
rm "$BIBMACROS"

# On Linux, rfrio is always expanded to {Rev. fr. inform. rech. op\'{e}r.} in $BIBENTRIES. The
# following line changes it back.
sed "s|\=[[:space:]]{Rev. fr. inform. rech. op\\\\'{e}r.}|\= rfrio|g" "$BIBENTRIES" >> "$OUTPUTBIB"

rm "$BIBENTRIES"


# Make a backup of $OUTPUTFILE under $BAKDIR if it exists and the trimmed file is different from
# the original $OUTPUTFILE.
FULLBIBNAME=${OUTPUTFILE//\//%}
BAKBIBFILE=$FULLBIBNAME-$TIME
BIBDIFFERENT=0
if [[ -f "$OUTPUTFILE" ]]; then
    printf "****** Difference between the trimmed bib and the original %s ******\n" "$OUTPUTFILE" > $TRIMBIBLOG
    # Remove the comments before comparing.
    diff <(grep -v "^[[:space:]]*%" "$OUTPUTBIB") <(grep -v "^[[:space:]]*%" "$OUTPUTFILE") >> $TRIMBIBLOG
    if [[ $? -eq 1 ]]; then
        BIBDIFFERENT=1
        # $BAKDIR/$FULLBIBNAME-latest_backup is always the latest backup.
        mkdir -p $BAKDIR
        cp "$OUTPUTFILE" "$BAKDIR/$FULLBIBNAME-latest_backup"
    	mv "$OUTPUTFILE" "$BAKDIR/$BAKBIBFILE"
        # Keep at most 50 latest files under $BAKDIR.
        (cd "$BAKDIR" && ls -tp | grep -v '/$' | grep "$FULLBIBNAME" | tail -n +51 | xargs -I {} rm -- {})
    fi
fi

# Write the output to $OUTPUTFILE
mv "$OUTPUTBIB" "$OUTPUTFILE"


# Check the bib file after trimming.
if [[ $BIBLATEX -eq 1 ]] ; then
    checkbib -l "$OUTPUTFILE"
else
    checkbib "$OUTPUTFILE"
fi

{
    printf "\n***************** checkbib.log after trimming ****************\n"
    cat $CHECKBIBLOG
    printf "\n"
} >> $CHECKBIBLOG.bak
mv $CHECKBIBLOG.bak $CHECKBIBLOG

cat "$CHECKBIBLOG"  | cat - "$TRIMBIBLOG" > "tmp-$TIME" && mv "tmp-$TIME" $TRIMBIBLOG
rm $CHECKBIBLOG

printf "Done. See %s for %s and the difference due to the trimming.\n" "$TRIMBIBLOG" "$CHECKBIBLOG"
if [[ $BIBDIFFERENT -eq 1 ]]; then
    printf "\nA copy of the original bib file is saved at %s" "$BAKDIR/$BAKBIBFILE"
    printf "\nA copy of the original bib file is saved at %s" "$BAKDIR/$BAKBIBFILE" >> $TRIMBIBLOG
fi

printf "\n"

exit 0