468 lines
13 KiB
Bash
Executable File
468 lines
13 KiB
Bash
Executable File
#!/usr/bin/env bash
|
||
set -Eeuo pipefail
|
||
|
||
AUDIT_VERSION=1
|
||
|
||
AUDIT_DIR="/data/Music/Beets/audit"
|
||
STAMP="$(date +%Y%m%d-%H%M%S)"
|
||
LOG="$AUDIT_DIR/library-audit-$STAMP.log"
|
||
|
||
AUDIT_DONE_FIELD="library_audited"
|
||
AUDIT_VERSION_FIELD="library_audit_version"
|
||
AUDIT_AT_FIELD="library_audited_at"
|
||
|
||
MODE="default"
|
||
MARK_AUDITED=1
|
||
|
||
usage() {
|
||
cat <<EOF
|
||
Usage: $(basename "$0") [options]
|
||
|
||
Options:
|
||
--reaudit
|
||
Audit unaudited files plus files audited by an older audit version.
|
||
|
||
--force-audit
|
||
Audit all files, ignoring existing audit markers.
|
||
|
||
--no-mark
|
||
Do not mark files as audited after the run.
|
||
|
||
-h, --help
|
||
Show this help.
|
||
|
||
Current script audit version: $AUDIT_VERSION
|
||
EOF
|
||
}
|
||
|
||
while [[ $# -gt 0 ]]; do
|
||
case "$1" in
|
||
--reaudit)
|
||
MODE="reaudit"
|
||
shift
|
||
;;
|
||
--force-audit)
|
||
MODE="force"
|
||
shift
|
||
;;
|
||
--no-mark)
|
||
MARK_AUDITED=0
|
||
shift
|
||
;;
|
||
-h|--help)
|
||
usage
|
||
exit 0
|
||
;;
|
||
*)
|
||
echo "ERROR: unknown option: $1" >&2
|
||
usage >&2
|
||
exit 2
|
||
;;
|
||
esac
|
||
done
|
||
|
||
mkdir -p "$AUDIT_DIR"
|
||
|
||
echo "Using beet: $(command -v beet)"
|
||
echo "Mode: $MODE"
|
||
echo "Audit version: $AUDIT_VERSION"
|
||
echo "Audit log: $LOG"
|
||
echo
|
||
|
||
prompt_yes() {
|
||
local prompt="$1"
|
||
local ans=""
|
||
|
||
if [[ ! -t 0 ]]; then
|
||
echo "Non-interactive shell; skipping prompt: $prompt"
|
||
return 1
|
||
fi
|
||
|
||
read -r -p "$prompt Type yes to continue: " ans
|
||
[[ "$ans" == "yes" ]]
|
||
}
|
||
|
||
run_scoped() {
|
||
local action="$1"
|
||
shift
|
||
|
||
local old_version_max=$((AUDIT_VERSION - 1))
|
||
|
||
case "$MODE" in
|
||
default)
|
||
"$action" "$@" "^${AUDIT_DONE_FIELD}:1"
|
||
;;
|
||
|
||
force)
|
||
if (( $# > 0 )); then
|
||
"$action" "$@"
|
||
else
|
||
"$action" 'path::.'
|
||
fi
|
||
;;
|
||
|
||
reaudit)
|
||
"$action" "$@" "^${AUDIT_DONE_FIELD}:1"
|
||
|
||
if (( old_version_max >= 0 )); then
|
||
"$action" "$@" "${AUDIT_DONE_FIELD}:1" "${AUDIT_VERSION_FIELD}:..${old_version_max}"
|
||
fi
|
||
;;
|
||
|
||
*)
|
||
echo "ERROR: invalid mode: $MODE" >&2
|
||
exit 2
|
||
;;
|
||
esac
|
||
}
|
||
|
||
beet_ls_action() {
|
||
beet ls "$@" -f "$CURRENT_FMT"
|
||
}
|
||
|
||
beet_refetch_action() {
|
||
beet lyrics -f "$@"
|
||
}
|
||
|
||
beet_normalize_instrumental_action() {
|
||
beet modify -y -M -w "$@" 'lyrics=[Instrumental]'
|
||
}
|
||
|
||
beet_mark_audited_action() {
|
||
local now
|
||
now="$(date +%s)"
|
||
|
||
beet modify -y -M -W "$@" \
|
||
"${AUDIT_DONE_FIELD}=1" \
|
||
"${AUDIT_VERSION_FIELD}=${AUDIT_VERSION}" \
|
||
"${AUDIT_AT_FIELD}=${now}"
|
||
}
|
||
|
||
run_refetch_check() {
|
||
local name="$1"
|
||
local description="$2"
|
||
shift 2
|
||
|
||
local tmp
|
||
tmp="$(mktemp)"
|
||
|
||
CURRENT_FMT=$'ID: $id\nCHECK: '"$name"$'\nAUDITED: $library_audited\nAUDIT_VERSION: $library_audit_version\nARTIST: $artist\nALBUM: $album\nTRACK: $track. $title\nPATH: $path\nLYRICS:\n$lyrics\n---'
|
||
|
||
echo
|
||
echo "======================================================================"
|
||
echo "$name"
|
||
echo "$description"
|
||
echo "Query terms:"
|
||
printf ' %q\n' "$@"
|
||
echo "----------------------------------------------------------------------"
|
||
|
||
run_scoped beet_ls_action "$@" | tee "$tmp" | tee -a "$LOG"
|
||
|
||
if [[ ! -s "$tmp" ]]; then
|
||
echo "No matches."
|
||
rm -f "$tmp"
|
||
return 0
|
||
fi
|
||
|
||
echo
|
||
if prompt_yes "Run: beet lyrics -f <same scoped query> for $name?"; then
|
||
echo "Running forced lyrics refetch for $name..."
|
||
run_scoped beet_refetch_action "$@" | tee -a "$LOG"
|
||
else
|
||
echo "Skipped fix for $name."
|
||
fi
|
||
|
||
rm -f "$tmp"
|
||
}
|
||
|
||
run_normalize_instrumental_check() {
|
||
local name="$1"
|
||
local description="$2"
|
||
shift 2
|
||
|
||
local tmp
|
||
tmp="$(mktemp)"
|
||
|
||
CURRENT_FMT=$'ID: $id\nCHECK: '"$name"$'\nAUDITED: $library_audited\nAUDIT_VERSION: $library_audit_version\nARTIST: $artist\nALBUM: $album\nTRACK: $track. $title\nPATH: $path\nCURRENT LYRICS:\n$lyrics\n---'
|
||
|
||
echo
|
||
echo "======================================================================"
|
||
echo "$name"
|
||
echo "$description"
|
||
echo "Query terms:"
|
||
printf ' %q\n' "$@"
|
||
echo "----------------------------------------------------------------------"
|
||
|
||
run_scoped beet_ls_action "$@" | tee "$tmp" | tee -a "$LOG"
|
||
|
||
if [[ ! -s "$tmp" ]]; then
|
||
echo "No matches."
|
||
rm -f "$tmp"
|
||
return 0
|
||
fi
|
||
|
||
echo
|
||
if prompt_yes "Normalize matches to lyrics=[Instrumental] for $name?"; then
|
||
echo "Running instrumental normalization for $name..."
|
||
run_scoped beet_normalize_instrumental_action "$@" | tee -a "$LOG"
|
||
else
|
||
echo "Skipped normalization for $name."
|
||
fi
|
||
|
||
rm -f "$tmp"
|
||
}
|
||
|
||
run_review_only_check() {
|
||
local name="$1"
|
||
local description="$2"
|
||
shift 2
|
||
|
||
local tmp
|
||
tmp="$(mktemp)"
|
||
|
||
CURRENT_FMT=$'ID: $id\nCHECK: '"$name"$'\nAUDITED: $library_audited\nAUDIT_VERSION: $library_audit_version\nARTIST: $artist\nALBUM: $album\nTRACK: $track. $title\nPATH: $path\nLYRICS:\n$lyrics\n---'
|
||
|
||
echo
|
||
echo "======================================================================"
|
||
echo "$name"
|
||
echo "$description"
|
||
echo "Query terms:"
|
||
printf ' %q\n' "$@"
|
||
echo "----------------------------------------------------------------------"
|
||
|
||
run_scoped beet_ls_action "$@" | tee "$tmp" | tee -a "$LOG"
|
||
|
||
if [[ ! -s "$tmp" ]]; then
|
||
echo "No matches."
|
||
else
|
||
echo
|
||
echo "Review-only bucket. No automatic fix offered."
|
||
fi
|
||
|
||
rm -f "$tmp"
|
||
}
|
||
|
||
echo "Starting library audit."
|
||
echo "Nothing is changed during checks unless you type exactly: yes"
|
||
echo "At the end, audited files are marked unless --no-mark was supplied."
|
||
echo
|
||
|
||
# ---------------------------------------------------------------------
|
||
# High-confidence broken lyrics / refetch buckets
|
||
# ---------------------------------------------------------------------
|
||
|
||
run_refetch_check \
|
||
"PMEDIA_GARBAGE" \
|
||
"Lyrics containing pmedia garbage/stubs." \
|
||
'lyrics::(?i)pmedia'
|
||
|
||
run_refetch_check \
|
||
"URL_ONLY" \
|
||
"Lyrics field is only a URL-like value with http(s) or www." \
|
||
'lyrics::(?i)^\s*(https?://|www\.)\S+\s*$'
|
||
|
||
run_refetch_check \
|
||
"TELEGRAM_STUB" \
|
||
"Telegram/t.me URL stubs, including values without http(s)." \
|
||
'lyrics::(?i)^\s*(https?://)?(www\.)?(t\.me|telegram\.me|telegram\.dog)/\S+\s*$'
|
||
|
||
run_refetch_check \
|
||
"BARE_DOMAIN_OR_URI" \
|
||
"Bare domains such as Dedenia.com, plus path-like domain stubs." \
|
||
'lyrics::(?i)\b[a-z0-9][a-z0-9-]{1,63}\.(com|net|org|edu|gov|io|co|me|ru|cn|jp|kr|uk|de|fr|it|es|nl|se|no|fi|pl|br|mx|ca|au|in|info|biz|xyz|site|online|club|music|tv|fm)\b(/[^\s\]\)]*)?'
|
||
|
||
run_refetch_check \
|
||
"EMAIL_ADDRESS" \
|
||
"Lyrics containing an email address." \
|
||
'lyrics::(?i)\b[a-z0-9._%+-]+@[a-z0-9.-]+\.[a-z]{2,}\b'
|
||
|
||
run_refetch_check \
|
||
"WEB_JUNK" \
|
||
"HTML, Cloudflare, CAPTCHA, 403/404, JavaScript-required, or similar web-junk pages." \
|
||
'lyrics::(?i)(<!doctype\s+html|<html\b|<body\b|<script\b|cloudflare|captcha|access denied|403 forbidden|404 not found|page not found|enable javascript|javascript required|service unavailable)'
|
||
|
||
run_refetch_check \
|
||
"WEB_BOILERPLATE" \
|
||
"Cookie/privacy/terms/login/subscription boilerplate." \
|
||
'lyrics::(?i)(cookie policy|privacy policy|terms of use|all rights reserved|dmca|advertisement|enable cookies|sign in to|log in to|create an account|subscribe to|you are being redirected)'
|
||
|
||
run_refetch_check \
|
||
"HTML_OR_ENTITY_RESIDUE" \
|
||
"HTML tags or HTML entities embedded in lyrics." \
|
||
'lyrics::(?i)(<[a-z!/][^>]{0,100}>|</[a-z][^>]{0,100}>| |&|"|'|<|>|&#[0-9]{2,6};|&#x[0-9a-f]{2,6};)'
|
||
|
||
run_refetch_check \
|
||
"FETCH_ERROR" \
|
||
"Fetch/API failure text stored as lyrics." \
|
||
'lyrics::(?i)^\s*(error|exception|request failed|connection failed|timeout|rate limit|too many requests|forbidden|unauthorized|not found)\b'
|
||
|
||
run_refetch_check \
|
||
"JSON_OR_API_BLOB" \
|
||
"JSON/API response-looking garbage." \
|
||
'lyrics::(?is)(^\s*\{.*\}\s*$|"[a-z_]*(error|message|status|lyrics|data)[a-z_]*"\s*:|^\s*\[(\s*\{.*\}\s*,?)+\s*\]\s*$)'
|
||
|
||
run_refetch_check \
|
||
"FILE_PATH_BLOB" \
|
||
"Local file path or file:// accidentally stored as lyrics." \
|
||
'lyrics::(?i)(file://|[a-z]:\\|/home/|/tmp/|/var/|/mnt/|/music/|/storage/)'
|
||
|
||
run_refetch_check \
|
||
"METADATA_BLOB" \
|
||
"Artist:/Album:/Title:/Track:/URL:/Source: style metadata stored as lyrics." \
|
||
'lyrics::(?im)^\s*(artist|album|title|track|url|source|lyrics)\s*:'
|
||
|
||
run_refetch_check \
|
||
"TIMESTAMP_ONLY" \
|
||
"Lyrics are only LRC timestamps and no lyric text." \
|
||
'lyrics::(?i)^\s*(\[[0-9]{1,2}:[0-9]{2}(\.[0-9]{1,3})?\]\s*)+$'
|
||
|
||
run_refetch_check \
|
||
"SHORT_TIMESTAMP_ONLY" \
|
||
"Short timestamp-only synced-lyrics junk." \
|
||
'lyrics::(?is)^\s*(\[[0-9]{1,2}:[0-9]{2}(\.[0-9]{1,3})?\]\s*){1,8}\s*$' \
|
||
'^lyrics::(?i)^\s*\[instrumental\]\s*$'
|
||
|
||
run_refetch_check \
|
||
"EMPTY_LYRICS" \
|
||
"Empty/whitespace lyrics. Since true instrumentals should be [Instrumental], these are likely missing." \
|
||
'lyrics::(?s)^\s*$'
|
||
|
||
# ---------------------------------------------------------------------
|
||
# Strict corruption checks
|
||
# Do not use (?i) here; case-insensitive mojibake detection catches real accented text.
|
||
# ---------------------------------------------------------------------
|
||
|
||
run_refetch_check \
|
||
"REPLACEMENT_CHAR" \
|
||
"Unicode replacement character, usually real decoding corruption." \
|
||
'lyrics::<3A>'
|
||
|
||
run_refetch_check \
|
||
"LATIN_MOJIBAKE" \
|
||
"Case-sensitive UTF-8-as-Latin1/Windows-1252 mojibake candidates." \
|
||
'lyrics::(Ã.|Â.)'
|
||
|
||
run_refetch_check \
|
||
"PUNCT_MOJIBAKE" \
|
||
"Curly quote/dash/ellipsis mojibake candidates." \
|
||
'lyrics::(’|“|â€|–|—|…|‘)'
|
||
|
||
run_refetch_check \
|
||
"EMOJI_MOJIBAKE" \
|
||
"Emoji mojibake candidates." \
|
||
'lyrics::ðŸ'
|
||
|
||
run_refetch_check \
|
||
"CONTROL_CHAR" \
|
||
"ASCII control characters other than normal newline/tab/carriage return." \
|
||
'lyrics::[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]'
|
||
|
||
run_refetch_check \
|
||
"INVISIBLE_UNICODE" \
|
||
"Zero-width spaces, bidi marks, BOMs, and similar hidden Unicode characters." \
|
||
'lyrics::[\u200B-\u200F\u202A-\u202E\u2060-\u206F\uFEFF]'
|
||
|
||
# ---------------------------------------------------------------------
|
||
# Review / maybe refetch buckets
|
||
# ---------------------------------------------------------------------
|
||
|
||
run_refetch_check \
|
||
"SHORT_1_80_NON_INSTRUMENTAL" \
|
||
"Very short lyrics, excluding exact [Instrumental]. Can be legitimate: skits, interludes, one-line vocal samples." \
|
||
'lyrics::(?is)^.{1,80}$' \
|
||
'^lyrics::(?i)^\s*\[instrumental\]\s*$'
|
||
|
||
run_refetch_check \
|
||
"SHORT_81_200_NON_INSTRUMENTAL" \
|
||
"Short lyrics, excluding exact [Instrumental]. More false positives expected." \
|
||
'lyrics::(?is)^.{81,200}$' \
|
||
'^lyrics::(?i)^\s*\[instrumental\]\s*$'
|
||
|
||
run_refetch_check \
|
||
"LONG_20K_PLUS" \
|
||
"Very long lyrics. Could be broken page dumps, but long synced lyrics/medleys can be legitimate." \
|
||
'lyrics::(?s)^.{20000,}$'
|
||
|
||
run_refetch_check \
|
||
"LONG_50K_PLUS" \
|
||
"Extremely long lyrics. Stronger broken-content candidate than 20K+." \
|
||
'lyrics::(?s)^.{50000,}$'
|
||
|
||
run_refetch_check \
|
||
"HUGE_UNBROKEN_TOKEN" \
|
||
"Huge unbroken tokens, often URLs/base64/minified HTML/JSON." \
|
||
'lyrics::(?s)\S{120,}'
|
||
|
||
run_refetch_check \
|
||
"LRC_METADATA" \
|
||
"LRC metadata tags such as [ar:], [al:], [ti:], [by:], [offset:]. Sometimes harmless." \
|
||
'lyrics::(?i)\[(ar|al|ti|by|offset|length|re|ve):[^\]]*\]'
|
||
|
||
run_refetch_check \
|
||
"SYMBOL_OR_PRIVATE_USE" \
|
||
"Private-use, box-drawing, and miscellaneous symbol ranges. Can be legitimate if lyrics contain emoji/symbols." \
|
||
'lyrics::[\uE000-\uF8FF\u2500-\u257F\u2600-\u27BF]'
|
||
|
||
# ---------------------------------------------------------------------
|
||
# Instrumental normalization buckets
|
||
# These should not use beet lyrics -f.
|
||
# ---------------------------------------------------------------------
|
||
|
||
run_normalize_instrumental_check \
|
||
"PLACEHOLDER_TO_INSTRUMENTAL" \
|
||
"Placeholder values that should become exactly [Instrumental], excluding already-correct values." \
|
||
'lyrics::(?i)^\s*\[?(instrumental|inst\.?|no lyrics|lyrics unavailable|lyrics not found|not available|n/?a|none|null|undefined|unknown)\]?\s*$' \
|
||
'^lyrics::(?i)^\s*\[instrumental\]\s*$'
|
||
|
||
run_normalize_instrumental_check \
|
||
"LRC_INSTRUMENTAL_TO_INSTRUMENTAL" \
|
||
"LRC timestamp-prefixed Instrumental values, e.g. [00:00.00]Instrumental." \
|
||
'lyrics::(?is)^\s*(\[[0-9]{1,2}:[0-9]{2}(\.[0-9]{1,3})?\]\s*)*instrumental\s*$' \
|
||
'^lyrics::(?i)^\s*\[instrumental\]\s*$'
|
||
|
||
run_normalize_instrumental_check \
|
||
"TITLE_INSTRUMENTAL_EMPTYISH_LYRICS" \
|
||
"Tracks with instrumental in the title and empty/underscore/dash timestamp lyrics." \
|
||
'title::(?i)\binstrumental\b' \
|
||
'lyrics::(?is)^\s*(\[[0-9]{1,2}:[0-9]{2}(\.[0-9]{1,3})?\]\s*)*[_\-.–—]*\s*$'
|
||
|
||
|
||
run_review_only_check \
|
||
"INSTRUMENTAL_PLUS_OTHER_TEXT" \
|
||
"Lyrics start with [Instrumental] but contain additional lyric text. Review manually; could be mostly-instrumental with outro/sample." \
|
||
'lyrics::(?is)^\s*\[instrumental\].+' \
|
||
'^lyrics::(?i)^\s*\[instrumental\]\s*$'
|
||
|
||
# ---------------------------------------------------------------------
|
||
# Mark audited
|
||
# ---------------------------------------------------------------------
|
||
|
||
echo
|
||
echo "======================================================================"
|
||
echo "Audit marker update"
|
||
echo "----------------------------------------------------------------------"
|
||
|
||
if (( MARK_AUDITED == 1 )); then
|
||
echo "This will mark the current audit scope with:"
|
||
echo " ${AUDIT_DONE_FIELD}=1"
|
||
echo " ${AUDIT_VERSION_FIELD}=${AUDIT_VERSION}"
|
||
echo " ${AUDIT_AT_FIELD}=<current unix timestamp>"
|
||
echo
|
||
|
||
echo "NOTE: this marks every item in the current audit scope as audited,"
|
||
echo "including files that did not match any dirty-lyrics check."
|
||
echo
|
||
|
||
if prompt_yes "Mark the full current audit scope as audited?"; then
|
||
run_scoped beet_mark_audited_action | tee -a "$LOG"
|
||
else
|
||
echo "Skipped audit marking."
|
||
fi
|
||
else
|
||
echo "Skipped audit marking due to --no-mark."
|
||
fi
|
||
|
||
echo
|
||
echo "Done."
|
||
echo "Audit log: $LOG"
|