From 4666e0456051edf22fc0bae331c1df874bcaf291 Mon Sep 17 00:00:00 2001 From: Taiki Endo Date: Tue, 4 Feb 2025 01:29:05 +0900 Subject: [PATCH] tools: Update tidy.sh and related configs --- .editorconfig | 4 +- .shellcheckrc | 27 +- tools/tidy.sh | 1202 ++++++++++++++++++++++++++++++++++++------------- 3 files changed, 922 insertions(+), 311 deletions(-) diff --git a/.editorconfig b/.editorconfig index 3aa2b380..98bc8985 100644 --- a/.editorconfig +++ b/.editorconfig @@ -11,12 +11,14 @@ indent_style = space insert_final_newline = true trim_trailing_whitespace = true -[*.{json,md,rb,sh,yml,yaml}] +[*.{css,html,json,md,rb,sh,yml,yaml}] indent_size = 2 [*.{js,yml,yaml}] quote_type = single [*.sh] +# https://google.github.io/styleguide/shellguide.html#s5.3-pipelines binary_next_line = true +# https://google.github.io/styleguide/shellguide.html#s5.5-case-statement switch_case_indent = true diff --git a/.shellcheckrc b/.shellcheckrc index 9de7d943..339847ea 100644 --- a/.shellcheckrc +++ b/.shellcheckrc @@ -2,13 +2,34 @@ # https://github.com/koalaman/shellcheck/blob/HEAD/shellcheck.1.md#rc-files # See also: -# https://www.shellcheck.net/wiki/Optional +# https://github.com/koalaman/shellcheck/wiki/Optional # https://google.github.io/styleguide/shellguide.html -# https://www.shellcheck.net/wiki/SC2292 +# https://github.com/koalaman/shellcheck/wiki/SC2249 +# enable=add-default-case + +# https://github.com/koalaman/shellcheck/wiki/SC2244 +enable=avoid-nullary-conditions + +# https://github.com/koalaman/shellcheck/wiki/SC2312 +# enable=check-extra-masked-returns + +# https://github.com/koalaman/shellcheck/wiki/SC2310 +# https://github.com/koalaman/shellcheck/wiki/SC2311 +# enable=check-set-e-suppressed + +# enable=check-unassigned-uppercase + +# https://github.com/koalaman/shellcheck/wiki/SC2230 +enable=deprecate-which + +# https://github.com/koalaman/shellcheck/wiki/SC2248 +enable=quote-safe-variables + +# https://github.com/koalaman/shellcheck/wiki/SC2292 # https://google.github.io/styleguide/shellguide.html#s6.3-tests enable=require-double-brackets -# https://www.shellcheck.net/wiki/SC2250 +# https://github.com/koalaman/shellcheck/wiki/SC2250 # https://google.github.io/styleguide/shellguide.html#s5.6-variable-expansion enable=require-variable-braces diff --git a/tools/tidy.sh b/tools/tidy.sh index 344c7fa8..54d41220 100755 --- a/tools/tidy.sh +++ b/tools/tidy.sh @@ -1,29 +1,63 @@ #!/usr/bin/env bash # SPDX-License-Identifier: Apache-2.0 OR MIT # shellcheck disable=SC2046 -set -eEuo pipefail +set -CeEuo pipefail IFS=$'\n\t' -cd "$(dirname "$0")"/.. - -# shellcheck disable=SC2154 -trap 's=$?; echo >&2 "$0: error on line "${LINENO}": ${BASH_COMMAND}"; exit ${s}' ERR +trap -- 's=$?; printf >&2 "%s\n" "${0##*/}:${LINENO}: \`${BASH_COMMAND}\` exit with ${s}"; exit ${s}' ERR +trap -- 'printf >&2 "%s\n" "${0##*/}: trapped SIGINT"; exit 1' SIGINT +cd -- "$(dirname -- "$0")"/.. # USAGE: # ./tools/tidy.sh # # Note: This script requires the following tools: +# - git +# - jq 1.6+ +# - npm (node 18+) +# - python 3.6+ # - shfmt # - shellcheck -# - npm -# - jq -# - python 3 -# - rustup (if Rust code exists) -# - clang-format (if C/C++ code exists) +# - cargo, rustfmt (if Rust code exists) +# - clang-format (if C/C++/Protobuf code exists) +# - parse-dockerfile (if Dockerfile exists) # # This script is shared with other repositories, so there may also be # checks for files not included in this repository, but they will be # skipped if the corresponding files do not exist. +retry() { + for i in {1..10}; do + if "$@"; then + return 0 + else + sleep "${i}" + fi + done + "$@" +} +error() { + if [[ -n "${GITHUB_ACTIONS:-}" ]]; then + printf '::error::%s\n' "$*" + else + printf >&2 'error: %s\n' "$*" + fi + should_fail=1 +} +warn() { + if [[ -n "${GITHUB_ACTIONS:-}" ]]; then + printf '::warning::%s\n' "$*" + else + printf >&2 'warning: %s\n' "$*" + fi +} +info() { + printf >&2 'info: %s\n' "$*" +} +print_fenced() { + printf '=======================================\n' + printf '%s' "$*" + printf '=======================================\n\n' +} check_diff() { if [[ -n "${CI:-}" ]]; then if ! git --no-pager diff --exit-code "$@"; then @@ -37,14 +71,14 @@ check_diff() { } check_config() { if [[ ! -e "$1" ]]; then - error "could not found $1 in the repository root" + error "could not found $1 in the repository root${2:-}" fi } check_install() { for tool in "$@"; do - if ! type -P "${tool}" &>/dev/null; then - if [[ "${tool}" == "python3" ]]; then - if type -P python &>/dev/null; then + if ! type -P "${tool}" >/dev/null; then + if [[ "${tool}" == 'python3' ]]; then + if type -P python >/dev/null; then continue fi fi @@ -53,41 +87,40 @@ check_install() { fi done } -info() { - echo >&2 "info: $*" -} -error() { - if [[ -n "${GITHUB_ACTIONS:-}" ]]; then - echo "::error::$*" - else - echo >&2 "error: $*" - fi - should_fail=1 -} -venv() { - local bin="$1" +check_unused() { + local kind="$1" shift - "${venv_bin}/${bin}${exe}" "$@" + local res + res=$(ls_files "$@") + if [[ -n "${res}" ]]; then + error "the following files are unused because there is no ${kind}; consider removing them" + print_fenced "${res}"$'\n' + fi +} +check_alt() { + local recommended=$1 + local not_recommended=$2 + if [[ -n "$3" ]]; then + error "please use ${recommended} instead of ${not_recommended} for consistency" + print_fenced "$3"$'\n' + fi +} +check_hidden() { + local res + for file in "$@"; do + check_alt ".${file}" "${file}" "$(comm -23 <(ls_files "*${file}") <(ls_files "*.${file}"))" + done +} +sed_rhs_escape() { + sed 's/\\/\\\\/g; s/\&/\\\&/g; s/\//\\\//g' <<<"$1" } venv_install_yq() { - local py_suffix='' - if type -P python3 &>/dev/null; then - py_suffix='3' - fi - exe='' - venv_bin='.venv/bin' - case "$(uname -s)" in - MINGW* | MSYS* | CYGWIN* | Windows_NT) - exe='.exe' - venv_bin='.venv/Scripts' - ;; - esac - if [[ ! -d .venv ]]; then - "python${py_suffix}" -m venv .venv - fi if [[ ! -e "${venv_bin}/yq${exe}" ]]; then - info "installing yq to ./.venv using pip" - venv "pip${py_suffix}" install yq + if [[ ! -d .venv ]]; then + "python${py_suffix}" -m venv .venv >&2 + fi + info "installing yq to .venv using pip${py_suffix}" + "${venv_bin}/pip${py_suffix}${exe}" install yq >&2 fi } @@ -99,280 +132,820 @@ EOF exit 1 fi +exe='' +py_suffix='' +if type -P python3 >/dev/null; then + py_suffix=3 +fi +venv_bin=.venv/bin +yq() { + venv_install_yq + "${venv_bin}/yq${exe}" "$@" +} +tomlq() { + venv_install_yq + "${venv_bin}/tomlq${exe}" "$@" +} +case "$(uname -s)" in + Linux) + if [[ "$(uname -o)" == 'Android' ]]; then + ostype=android + else + ostype=linux + fi + ;; + Darwin) ostype=macos ;; + FreeBSD) ostype=freebsd ;; + NetBSD) ostype=netbsd ;; + OpenBSD) ostype=openbsd ;; + DragonFly) ostype=dragonfly ;; + SunOS) + if [[ "$(/usr/bin/uname -o)" == 'illumos' ]]; then + ostype=illumos + else + ostype=solaris + # Solaris /usr/bin/* are not POSIX-compliant (e.g., grep has no -q, -E, -F), + # and POSIX-compliant commands are in /usr/xpg{4,6,7}/bin. + # https://docs.oracle.com/cd/E88353_01/html/E37853/xpg-7.html + if [[ "${PATH}" != *'/usr/xpg4/bin'* ]]; then + export PATH="/usr/xpg4/bin:${PATH}" + fi + # GNU/BSD grep/sed is required to run some checks, but most checks are okay with other POSIX grep/sed. + # Solaris /usr/xpg4/bin/grep has -q, -E, -F, but no -o (non-POSIX). + # Solaris /usr/xpg4/bin/sed has no -E (POSIX.1-2024) yet. + for tool in sed grep; do + if type -P "g${tool}" >/dev/null; then + eval "${tool}() { g${tool} \"\$@\"; }" + fi + done + fi + ;; + MINGW* | MSYS* | CYGWIN* | Windows_NT) + ostype=windows + exe=.exe + venv_bin=.venv/Scripts + if type -P jq >/dev/null; then + # https://github.com/jqlang/jq/issues/1854 + _tmp=$(jq -r .a <<<'{}') + if [[ "${_tmp}" != 'null' ]]; then + _tmp=$(jq -b -r .a 2>/dev/null <<<'{}' || true) + if [[ "${_tmp}" == 'null' ]]; then + jq() { command jq -b "$@"; } + else + jq() { command jq "$@" | tr -d '\r'; } + fi + yq() { + venv_install_yq + "${venv_bin}/yq${exe}" "$@" | tr -d '\r' + } + tomlq() { + venv_install_yq + "${venv_bin}/tomlq${exe}" "$@" | tr -d '\r' + } + fi + fi + ;; + *) error "unrecognized os type '$(uname -s)' for \`\$(uname -s)\`" ;; +esac + +check_install git +exclude_from_ls_files=() +# - `find` lists symlinks. `! ( -name -prune )` (.i.e., ignore ) are manually listed from .gitignore. +# - `git submodule status` lists submodules. Use sed to remove the first character indicates status ( |+|-). +# - `git ls-files --deleted` lists removed files. +while IFS=$'\n' read -r line; do exclude_from_ls_files+=("${line}"); done < <({ + find . \! \( -name .git -prune \) \! \( -name target -prune \) \! \( -name .venv -prune \) \! \( -name tmp -prune \) -type l | cut -c3- + git submodule status | sed 's/^.//' | cut -d' ' -f2 + git ls-files --deleted +} | LC_ALL=C sort -u) +exclude_from_ls_files_no_symlink=() +while IFS=$'\n' read -r line; do exclude_from_ls_files_no_symlink+=("${line}"); done < <({ + git submodule status | sed 's/^.//' | cut -d' ' -f2 + git ls-files --deleted +} | LC_ALL=C sort -u) +ls_files() { + if [[ "${1:-}" == '--include-symlink' ]]; then + shift + comm -23 <(git ls-files "$@" | LC_ALL=C sort) <(printf '%s\n' ${exclude_from_ls_files_no_symlink[@]+"${exclude_from_ls_files_no_symlink[@]}"}) + else + comm -23 <(git ls-files "$@" | LC_ALL=C sort) <(printf '%s\n' ${exclude_from_ls_files[@]+"${exclude_from_ls_files[@]}"}) + fi +} + # Rust (if exists) -if [[ -n "$(git ls-files '*.rs')" ]]; then +if [[ -n "$(ls_files '*.rs')" ]]; then info "checking Rust code style" - check_install cargo jq python3 - check_config .rustfmt.toml - if check_install rustup; then + check_config .rustfmt.toml "; consider adding with reference to https://github.com/taiki-e/cargo-hack/blob/HEAD/.rustfmt.toml" + check_config .clippy.toml "; consider adding with reference to https://github.com/taiki-e/cargo-hack/blob/HEAD/.clippy.toml" + if check_install cargo jq python3; then # `cargo fmt` cannot recognize files not included in the current workspace and modules # defined inside macros, so run rustfmt directly. # We need to use nightly rustfmt because we use the unstable formatting options of rustfmt. - rustc_version=$(rustc -vV | grep '^release:' | cut -d' ' -f2) - if [[ "${rustc_version}" == *"nightly"* ]] || [[ "${rustc_version}" == *"dev"* ]]; then - rustup component add rustfmt &>/dev/null + rustc_version=$(rustc -vV | grep -E '^release:' | cut -d' ' -f2) + if [[ "${rustc_version}" =~ nightly|dev ]] || ! type -P rustup >/dev/null; then + if type -P rustup >/dev/null; then + retry rustup component add rustfmt &>/dev/null + fi info "running \`rustfmt \$(git ls-files '*.rs')\`" - rustfmt $(git ls-files '*.rs') + rustfmt $(ls_files '*.rs') else - rustup component add rustfmt --toolchain nightly &>/dev/null + if type -P rustup >/dev/null; then + retry rustup component add rustfmt --toolchain nightly &>/dev/null + fi info "running \`rustfmt +nightly \$(git ls-files '*.rs')\`" - rustfmt +nightly $(git ls-files '*.rs') + rustfmt +nightly $(ls_files '*.rs') fi - check_diff $(git ls-files '*.rs') - fi - cast_without_turbofish=$(grep -n -E '\.cast\(\)' $(git ls-files '*.rs') || true) - if [[ -n "${cast_without_turbofish}" ]]; then - error "please replace \`.cast()\` with \`.cast::()\`:" - echo "${cast_without_turbofish}" - fi - # Sync readme and crate-level doc. - first='1' - for readme in $(git ls-files '*README.md'); do - if ! grep -q '^' "${readme}"; then - continue + check_diff $(ls_files '*.rs') + cast_without_turbofish=$(grep -Fn '.cast()' $(ls_files '*.rs') || true) + if [[ -n "${cast_without_turbofish}" ]]; then + error "please replace \`.cast()\` with \`.cast::()\`:" + printf '%s\n' "${cast_without_turbofish}" fi - lib="$(dirname "${readme}")/src/lib.rs" - if [[ -n "${first}" ]]; then - first='' - info "checking readme and crate-level doc are synchronized" - fi - if ! grep -q '^' "${readme}"; then - bail "missing '' comment in ${readme}" - fi - if ! grep -q '^' "${lib}"; then - bail "missing '' comment in ${lib}" - fi - if ! grep -q '^' "${lib}"; then - bail "missing '' comment in ${lib}" - fi - new=$(tr <"${readme}" '\n' '\a' | grep -o '.*' | sed 's/\&/\\\&/g; s/\\/\\\\/g') - new=$(tr <"${lib}" '\n' '\a' | awk -v new="${new}" 'gsub(".*",new)' | tr '\a' '\n') - echo "${new}" >"${lib}" - check_diff "${lib}" - done - # Make sure that public Rust crates don't contain executables and binaries. - executables='' - binaries='' - metadata=$(cargo metadata --format-version=1 --no-deps) - has_public_crate='' - has_root_crate='' - venv_install_yq - for id in $(jq <<<"${metadata}" '.workspace_members[]'); do - pkg=$(jq <<<"${metadata}" ".packages[] | select(.id == ${id})") - publish=$(jq <<<"${pkg}" -r '.publish') - manifest_path=$(jq <<<"${pkg}" -r '.manifest_path') - if [[ "$(venv tomlq -c '.lints' "${manifest_path}")" == "null" ]]; then - error "no [lints] table in ${manifest_path} please add '[lints]' with 'workspace = true'" - fi - # Publishing is unrestricted if null, and forbidden if an empty array. - if [[ "${publish}" == "[]" ]]; then - continue - fi - has_public_crate='1' - done - if [[ -n "${has_public_crate}" ]]; then - info "checking public crates don't contain executables and binaries" + # Make sure that public Rust crates don't contain executables and binaries. + executables='' + binaries='' + metadata=$(cargo metadata --format-version=1 --no-deps) + root_manifest='' if [[ -f Cargo.toml ]]; then root_manifest=$(cargo locate-project --message-format=plain --manifest-path Cargo.toml) - root_pkg=$(jq <<<"${metadata}" ".packages[] | select(.manifest_path == \"${root_manifest}\")") - if [[ -n "${root_pkg}" ]]; then - publish=$(jq <<<"${root_pkg}" -r '.publish') - # Publishing is unrestricted if null, and forbidden if an empty array. - if [[ "${publish}" != "[]" ]]; then - has_root_crate=1 - exclude=$(venv tomlq -r '.package.exclude[]' Cargo.toml) - if ! grep <<<"${exclude}" -Eq '^/\.\*$'; then - error "top-level Cargo.toml of non-virtual workspace should have 'exclude' field with \"/.*\"" - fi - if [[ -e tools ]] && ! grep <<<"${exclude}" -Eq '^/tools$'; then - error "top-level Cargo.toml of non-virtual workspace should have 'exclude' field with \"/tools\" if it exists" - fi - if [[ -e target-specs ]] && ! grep <<<"${exclude}" -Eq '^/target-specs$'; then - error "top-level Cargo.toml of non-virtual workspace should have 'exclude' field with \"/target-specs\" if it exists" - fi - fi - fi fi - for p in $(git ls-files); do - # Skip directories. - if [[ -d "${p}" ]]; then + exclude='' + has_public_crate='' + has_root_crate='' + for pkg in $(jq -c '. as $metadata | .workspace_members[] as $id | $metadata.packages[] | select(.id == $id)' <<<"${metadata}"); do + eval "$(jq -r '@sh "publish=\(.publish) manifest_path=\(.manifest_path)"' <<<"${pkg}")" + if [[ "$(tomlq -c '.lints' "${manifest_path}")" == 'null' ]]; then + error "no [lints] table in ${manifest_path} please add '[lints]' with 'workspace = true'" + fi + # Publishing is unrestricted if null, and forbidden if an empty array. + if [[ -z "${publish}" ]]; then continue fi - # Top-level hidden files/directories and tools/* are excluded from crates.io (ensured by the above check). - # TODO: fully respect exclude field in Cargo.toml. - case "${p}" in - .* | tools/* | target-specs/*) continue ;; - */*) ;; - *) - # If there is no crate at root, executables at the repository root directory if always okay. - if [[ -z "${has_root_crate}" ]]; then - continue - fi - ;; - esac - if [[ -x "${p}" ]]; then - executables+="${p}"$'\n' - fi - # Use diff instead of file because file treats an empty file as a binary - # https://unix.stackexchange.com/questions/275516/is-there-a-convenient-way-to-classify-files-as-binary-or-text#answer-402870 - if (diff .gitattributes "${p}" || true) | grep -q '^Binary file'; then - binaries+="${p}"$'\n' + has_public_crate=1 + if [[ "${manifest_path}" == "${root_manifest}" ]]; then + has_root_crate=1 + exclude=$(tomlq -r '.package.exclude[]' "${manifest_path}") + if ! grep -Eq '^/\.\*$' <<<"${exclude}"; then + error "top-level Cargo.toml of non-virtual workspace should have 'exclude' field with \"/.*\"" + fi + if [[ -e tools ]] && ! grep -Eq '^/tools$' <<<"${exclude}"; then + error "top-level Cargo.toml of non-virtual workspace should have 'exclude' field with \"/tools\" if it exists" + fi + if [[ -e target-specs ]] && ! grep -Eq '^/target-specs$' <<<"${exclude}"; then + error "top-level Cargo.toml of non-virtual workspace should have 'exclude' field with \"/target-specs\" if it exists" + fi fi done - if [[ -n "${executables}" ]]; then - error "file-permissions-check failed: executables are only allowed to be present in directories that are excluded from crates.io" - echo "=======================================" - echo -n "${executables}" - echo "=======================================" - fi - if [[ -n "${binaries}" ]]; then - error "file-permissions-check failed: binaries are only allowed to be present in directories that are excluded from crates.io" - echo "=======================================" - echo -n "${binaries}" - echo "=======================================" + if [[ -n "${has_public_crate}" ]]; then + check_config .deny.toml "; consider adding with reference to https://github.com/taiki-e/cargo-hack/blob/HEAD/.deny.toml" + info "checking public crates don't contain executables and binaries" + for p in $(ls_files --include-symlink); do + # Skip directories. + if [[ -d "${p}" ]]; then + continue + fi + # Top-level hidden files/directories and tools/* are excluded from crates.io (ensured by the above check). + # TODO: fully respect exclude field in Cargo.toml. + case "${p}" in + .* | tools/* | target-specs/*) continue ;; + */*) ;; + *) + # If there is no crate at root, executables at the repository root directory if always okay. + if [[ -z "${has_root_crate}" ]]; then + continue + fi + ;; + esac + if [[ -x "${p}" ]]; then + executables+="${p}"$'\n' + fi + # Use `diff` instead of `file` because `file` treats an empty file as a binary. + # https://unix.stackexchange.com/questions/275516/is-there-a-convenient-way-to-classify-files-as-binary-or-text#answer-402870 + if { diff .gitattributes "${p}" || true; } | grep -Eq '^Binary file'; then + binaries+="${p}"$'\n' + fi + done + if [[ -n "${executables}" ]]; then + error "file-permissions-check failed: executables are only allowed to be present in directories that are excluded from crates.io" + print_fenced "${executables}" + fi + if [[ -n "${binaries}" ]]; then + error "file-permissions-check failed: binaries are only allowed to be present in directories that are excluded from crates.io" + print_fenced "${binaries}" + fi fi fi -elif [[ -e .rustfmt.toml ]]; then - error ".rustfmt.toml is unused" + # Sync markdown to rustdoc. + first=1 + for markdown in $(ls_files '*.md'); do + markers=$(grep -En '^' "${markdown}" || true) + # BSD wc's -l emits spaces before number. + if [[ ! "$(LC_ALL=C wc -l <<<"${markers}")" =~ ^\ *2$ ]]; then + if [[ -n "${markers}" ]]; then + error "inconsistent '' marker found in ${markdown}" + printf '%s\n' "${markers}" + fi + continue + fi + start_marker=$(head -n1 <<<"${markers}") + end_marker=$(head -n2 <<<"${markers}" | tail -n1) + if [[ "${start_marker}" == *"tidy:sync-markdown-to-rustdoc:end"* ]] || [[ "${end_marker}" == *"tidy:sync-markdown-to-rustdoc:start"* ]]; then + error "inconsistent '' marker found in ${markdown}" + printf '%s\n' "${markers}" + continue + fi + if [[ -n "${first}" ]]; then + first='' + info "syncing markdown to rustdoc" + fi + lib="${start_marker#*:<\!-- tidy:sync-markdown-to-rustdoc:start:}" + if [[ "${start_marker}" == "${lib}" ]]; then + error "missing path in '' marker in ${markdown}" + printf '%s\n' "${markers}" + continue + fi + lib="${lib% -->}" + lib="$(dirname -- "${markdown}")/${lib}" + markers=$(grep -En '^' "${lib}" || true) + # BSD wc's -l emits spaces before number. + if [[ ! "$(LC_ALL=C wc -l <<<"${markers}")" =~ ^\ *2$ ]]; then + if [[ -n "${markers}" ]]; then + error "inconsistent '' marker found in ${lib}" + printf '%s\n' "${markers}" + else + error "missing '' marker in ${lib}" + fi + continue + fi + start_marker=$(head -n1 <<<"${markers}") + end_marker=$(head -n2 <<<"${markers}" | tail -n1) + if [[ "${start_marker}" == *"tidy:sync-markdown-to-rustdoc:end"* ]] || [[ "${end_marker}" == *"tidy:sync-markdown-to-rustdoc:start"* ]]; then + error "inconsistent '' marker found in ${lib}" + printf '%s\n' "${markers}" + continue + fi + new=''$'\a' + empty_line_re='^ *$' + gfm_alert_re='^> {0,4}\[!.*\] *$' + rust_code_block_re='^ *```(rust|rs) *$' + code_block_attr='' + in_alert='' + first_line=1 + ignore='' + while IFS='' read -rd$'\a' line; do + if [[ -n "${ignore}" ]]; then + if [[ "${line}" == ''* ]]; then + ignore='' + fi + continue + fi + if [[ -n "${first_line}" ]]; then + # Ignore start marker. + first_line='' + continue + elif [[ -n "${in_alert}" ]]; then + if [[ "${line}" =~ ${empty_line_re} ]]; then + in_alert='' + new+=$'\a'""$'\a' + fi + elif [[ "${line}" =~ ${gfm_alert_re} ]]; then + alert="${line#*[\!}" + alert="${alert%%]*}" + alert=$(tr '[:lower:]' '[:upper:]' <<<"${alert%%]*}") + alert_lower=$(tr '[:upper:]' '[:lower:]' <<<"${alert}") + case "${alert}" in + NOTE | TIP | IMPORTANT) alert_sign='ⓘ' ;; + WARNING | CAUTION) alert_sign='⚠' ;; + *) + error "unknown alert type '${alert}' found; please use one of the types listed in " + new+="${line}"$'\a' + continue + ;; + esac + in_alert=1 + new+="
"$'\a\a' + new+="> **${alert_sign} ${alert:0:1}${alert_lower:1}**"$'\a>\a' + continue + fi + if [[ "${line}" =~ ${rust_code_block_re} ]]; then + code_block_attr="${code_block_attr#<\!-- tidy:sync-markdown-to-rustdoc:code-block:}" + code_block_attr="${code_block_attr%% -->*}" + new+="${line/\`\`\`*/\`\`\`}${code_block_attr}"$'\a' + code_block_attr='' + continue + fi + if [[ -n "${code_block_attr}" ]]; then + error "'${code_block_attr}' ignored because there is no subsequent Rust code block" + code_block_attr='' + fi + if [[ "${line}" == ''* ]]; then + code_block_attr="${line}" + continue + fi + if [[ "${line}" == ''* ]]; then + if [[ "${new}" == *$'\a\a' ]]; then + new="${new%$'\a'}" + fi + ignore=1 + continue + fi + new+="${line}"$'\a' + done < <(tr '\n' '\a' <"${markdown}" | grep -Eo '.*') + new+='' + new=$(tr '\n' '\a' <"${lib}" | sed "s/.*/$(sed_rhs_escape "${new}")/" | tr '\a' '\n') + printf '%s\n' "${new}" >|"${lib}" + check_diff "${lib}" + done + printf '\n' +else + check_unused "Rust code" '*.cargo*' '*clippy.toml' '*deny.toml' '*rustfmt.toml' '*Cargo.toml' '*Cargo.lock' fi +check_hidden clippy.toml deny.toml rustfmt.toml -# C/C++ (if exists) -if [[ -n "$(git ls-files '*.c' '*.h' '*.cpp' '*.hpp')" ]]; then - info "checking C/C++ code style" +# C/C++/Protobuf (if exists) +clang_format_ext=('*.c' '*.h' '*.cpp' '*.hpp' '*.proto') +if [[ -n "$(ls_files "${clang_format_ext[@]}")" ]]; then + info "checking C/C++/Protobuf code style" check_config .clang-format if check_install clang-format; then - info "running \`clang-format -i \$(git ls-files '*.c' '*.h' '*.cpp' '*.hpp')\`" - clang-format -i $(git ls-files '*.c' '*.h' '*.cpp' '*.hpp') - check_diff $(git ls-files '*.c' '*.h' '*.cpp' '*.hpp') + IFS=' ' + info "running \`clang-format -i \$(git ls-files ${clang_format_ext[*]})\`" + IFS=$'\n\t' + clang-format -i $(ls_files "${clang_format_ext[@]}") + check_diff $(ls_files "${clang_format_ext[@]}") fi -elif [[ -e .clang-format ]]; then - error ".clang-format is unused" + printf '\n' +else + check_unused "C/C++/Protobuf code" '*.clang-format*' fi +check_alt '.clang-format' '_clang-format' "$(ls_files '*_clang-format')" +# https://gcc.gnu.org/onlinedocs/gcc/Overall-Options.html +check_alt '.cpp extension' 'other extensions' "$(ls_files '*.cc' '*.cp' '*.cxx' '*.C' '*.CPP' '*.c++')" +check_alt '.hpp extension' 'other extensions' "$(ls_files '*.hh' '*.hp' '*.hxx' '*.H' '*.HPP' '*.h++')" -# YAML/JavaScript/JSON (if exists) -if [[ -n "$(git ls-files '*.yml' '*.yaml' '*.js' '*.json')" ]]; then - info "checking YAML/JavaScript/JSON code style" +# YAML/HTML/CSS/JavaScript/JSON (if exists) +prettier_ext=('*.css' '*.html' '*.js' '*.json' '*.yml' '*.yaml') +if [[ -n "$(ls_files "${prettier_ext[@]}")" ]]; then + info "checking YAML/HTML/CSS/JavaScript/JSON code style" check_config .editorconfig - if check_install npm; then - info "running \`npx -y prettier -l -w \$(git ls-files '*.yml' '*.yaml' '*.js' '*.json')\`" - npx -y prettier -l -w $(git ls-files '*.yml' '*.yaml' '*.js' '*.json') - check_diff $(git ls-files '*.yml' '*.yaml' '*.js' '*.json') + if [[ "${ostype}" == 'solaris' ]] && [[ -n "${CI:-}" ]] && ! type -P npm >/dev/null; then + warn "this check is skipped on Solaris due to no node 18+ in upstream package manager" + elif check_install npm; then + IFS=' ' + info "running \`npx -y prettier -l -w \$(git ls-files ${prettier_ext[*]})\`" + IFS=$'\n\t' + npx -y prettier -l -w $(ls_files "${prettier_ext[@]}") + check_diff $(ls_files "${prettier_ext[@]}") fi - # Check GitHub workflows. - if [[ -d .github/workflows ]]; then - info "checking GitHub workflows" - if check_install jq python3; then - venv_install_yq - for workflow in .github/workflows/*.yml; do + printf '\n' +else + check_unused "YAML/HTML/CSS/JavaScript/JSON file" '*.prettierignore' +fi +# https://prettier.io/docs/en/configuration +check_alt '.editorconfig' 'other configs' "$(ls_files '*.prettierrc*' '*prettier.config.*')" +check_alt '.yml extension' '.yaml extension' "$(ls_files '*.yaml' | { grep -Fv '.markdownlint-cli2.yaml' || true; })" + +# TOML (if exists) +if [[ -n "$(ls_files '*.toml' | { grep -Fv '.taplo.toml' || true; })" ]]; then + info "checking TOML style" + check_config .taplo.toml + if [[ "${ostype}" == 'solaris' ]] && [[ -n "${CI:-}" ]] && ! type -P npm >/dev/null; then + warn "this check is skipped on Solaris due to no node 18+ in upstream package manager" + elif check_install npm; then + info "running \`npx -y @taplo/cli fmt \$(git ls-files '*.toml')\`" + RUST_LOG=warn npx -y @taplo/cli fmt $(ls_files '*.toml') + check_diff $(ls_files '*.toml') + fi + printf '\n' +else + check_unused "TOML file" '*taplo.toml' +fi +check_hidden taplo.toml + +# Markdown (if exists) +if [[ -n "$(ls_files '*.md')" ]]; then + info "checking markdown style" + check_config .markdownlint-cli2.yaml + if [[ "${ostype}" == 'solaris' ]] && [[ -n "${CI:-}" ]] && ! type -P npm >/dev/null; then + warn "this check is skipped on Solaris due to no node 18+ in upstream package manager" + elif check_install npm; then + info "running \`npx -y markdownlint-cli2 \$(git ls-files '*.md')\`" + if ! npx -y markdownlint-cli2 $(ls_files '*.md'); then + error "check failed; please resolve the above markdownlint error(s)" + fi + fi + printf '\n' +else + check_unused "markdown file" '*.markdownlint-cli2.yaml' +fi +# https://github.com/DavidAnson/markdownlint-cli2#configuration +check_alt '.markdownlint-cli2.yaml' 'other configs' "$(ls_files '*.markdownlint-cli2.jsonc' '*.markdownlint-cli2.cjs' '*.markdownlint-cli2.mjs' '*.markdownlint.*')" +check_alt '.md extension' '*.markdown extension' "$(ls_files '*.markdown')" + +# Shell scripts +info "checking shell scripts" +shell_files=() +docker_files=() +bash_files=() +grep_ere_files=() +sed_ere_files=() +for p in $(ls_files '*.sh' '*Dockerfile*'); do + case "${p}" in + tests/fixtures/* | */tests/fixtures/* | *.json) continue ;; + esac + case "${p##*/}" in + *.sh) + shell_files+=("${p}") + re='^#!/.*bash' + if [[ "$(head -1 "${p}")" =~ ${re} ]]; then + bash_files+=("${p}") + fi + ;; + *Dockerfile*) + docker_files+=("${p}") + bash_files+=("${p}") # TODO + ;; + esac + if grep -Eq '(^|[^0-9A-Za-z\."'\''-])(grep) -[A-Za-z]*E[^\)]' "${p}"; then + grep_ere_files+=("${p}") + fi + if grep -Eq '(^|[^0-9A-Za-z\."'\''-])(sed) -[A-Za-z]*E[^\)]' "${p}"; then + sed_ere_files+=("${p}") + fi +done +workflows=() +actions=() +if [[ -d .github/workflows ]]; then + for p in .github/workflows/*.yml; do + workflows+=("${p}") + bash_files+=("${p}") # TODO + done +fi +if [[ -n "$(ls_files '*action.yml')" ]]; then + for p in $(ls_files '*action.yml'); do + if [[ "${p##*/}" == 'action.yml' ]]; then + actions+=("${p}") + if ! grep -Fq 'shell: sh' "${p}"; then + bash_files+=("${p}") + fi + fi + done +fi +# correctness +res=$({ grep -En '(\[\[ .* ]]|(^|[^\$])\(\(.*\)\))( +#| *$)' "${bash_files[@]}" || true; } | { grep -Ev '^[^ ]+: *(#|//)' || true; } | LC_ALL=C sort) +if [[ -n "${res}" ]]; then + error "bare [[ ]] and (( )) may not work as intended: see https://github.com/koalaman/shellcheck/issues/2360 for more" + print_fenced "${res}"$'\n' +fi +# TODO: chmod|chown +res=$({ grep -En '(^|[^0-9A-Za-z\."'\''-])(basename|cat|cd|cp|dirname|ln|ls|mkdir|mv|pushd|rm|rmdir|tee|touch)( +-[0-9A-Za-z]+)* +[^<>\|-]' "${bash_files[@]}" || true; } | { grep -Ev '^[^ ]+: *(#|//)' || true; } | LC_ALL=C sort) +if [[ -n "${res}" ]]; then + error "use \`--\` before path(s): see https://github.com/koalaman/shellcheck/issues/2707 / https://github.com/koalaman/shellcheck/issues/2612 / https://github.com/koalaman/shellcheck/issues/2305 / https://github.com/koalaman/shellcheck/issues/2157 / https://github.com/koalaman/shellcheck/issues/2121 / https://github.com/koalaman/shellcheck/issues/314 for more" + print_fenced "${res}"$'\n' +fi +res=$({ grep -En '(^|[^0-9A-Za-z\."'\''-])(LINES|RANDOM|PWD)=' "${bash_files[@]}" || true; } | { grep -Ev '^[^ ]+: *(#|//)' || true; } | LC_ALL=C sort) +if [[ -n "${res}" ]]; then + error "do not modify these built-in bash variables: see https://github.com/koalaman/shellcheck/issues/2160 / https://github.com/koalaman/shellcheck/issues/2559 for more" + print_fenced "${res}"$'\n' +fi +# perf +res=$({ grep -En '(^|[^\\])\$\((cat) ' "${bash_files[@]}" || true; } | { grep -Ev '^[^ ]+: *(#|//)' || true; } | LC_ALL=C sort) +if [[ -n "${res}" ]]; then + error "use faster \`\$(' "${bash_files[@]}" || true; } | { grep -Ev '^[^ ]+: *(#|//)' || true; } | LC_ALL=C sort) +if [[ -n "${res}" ]]; then + error "\`type -P\` doesn't output to stderr; use \`>\` instead of \`&>\`" + print_fenced "${res}"$'\n' +fi +# TODO: multi-line case +res=$({ grep -En '(^|[^0-9A-Za-z\."'\''-])(echo|printf )[^;)]* \|[^\|]' "${bash_files[@]}" || true; } | { grep -Ev '^[^ ]+: *(#|//)' || true; } | LC_ALL=C sort) +if [[ -n "${res}" ]]; then + error "use faster \`<<<...\` instead of \`echo ... |\`/\`printf ... |\`: see https://github.com/koalaman/shellcheck/issues/2593 for more" + print_fenced "${res}"$'\n' +fi +# style +if [[ ${#grep_ere_files[@]} -gt 0 ]]; then + # We intentionally do not check for occurrences in any other order (e.g., -iE, -i -E) here. + # This enforces the style and makes it easier to search. + res=$({ grep -En '(^|[^0-9A-Za-z\."'\''-])(grep) +([^-]|-[^EFP-]|--[^hv])' "${grep_ere_files[@]}" || true; } | { grep -Ev '^[^ ]+: *(#|//)' || true; } | LC_ALL=C sort) + if [[ -n "${res}" ]]; then + error "please always use ERE (grep -E) instead of BRE for code consistency within a file" + print_fenced "${res}"$'\n' + fi +fi +if [[ ${#sed_ere_files[@]} -gt 0 ]]; then + res=$({ grep -En '(^|[^0-9A-Za-z\."'\''-])(sed) +([^-]|-[^E-]|--[^hv])' "${sed_ere_files[@]}" || true; } | { grep -Ev '^[^ ]+: *(#|//)' || true; } | LC_ALL=C sort) + if [[ -n "${res}" ]]; then + error "please always use ERE (sed -E) instead of BRE for code consistency within a file" + print_fenced "${res}"$'\n' + fi +fi +if check_install shfmt; then + check_config .editorconfig + info "running \`shfmt -w \$(git ls-files '*.sh')\`" + if ! shfmt -w "${shell_files[@]}"; then + error "check failed; please resolve the shfmt error(s)" + fi + check_diff "${shell_files[@]}" +fi +if [[ "${ostype}" == 'solaris' ]] && [[ -n "${CI:-}" ]] && ! type -P shellcheck >/dev/null; then + warn "this check is skipped on Solaris due to no haskell/shellcheck in upstream package manager" +elif check_install shellcheck; then + check_config .shellcheckrc + info "running \`shellcheck \$(git ls-files '*.sh')\`" + if ! shellcheck "${shell_files[@]}"; then + error "check failed; please resolve the above shellcheck error(s)" + fi + # Check scripts in dockerfile. + if [[ ${#docker_files[@]} -gt 0 ]]; then + # Exclude SC2096 due to the way the temporary script is created. + shellcheck_exclude=SC2096 + info "running \`shellcheck --exclude ${shellcheck_exclude}\` for scripts in \$(git ls-files '*Dockerfile*')\`" + if [[ "${ostype}" == 'windows' ]]; then + # No such file or directory: '/proc/N/fd/N' + warn "this check is skipped on Windows due to upstream bug (failed to found fd created by <())" + elif [[ "${ostype}" == 'dragonfly' ]]; then + warn "this check is skipped on DragonFly BSD due to upstream bug (hang)" + elif check_install jq python3 parse-dockerfile; then + shellcheck_for_dockerfile() { + local text=$1 + local shell=$2 + local display_path=$3 + if [[ "${text}" == 'null' ]]; then + return + fi + text="#!${shell}"$'\n'"${text}" + case "${ostype}" in + windows) text=${text//\r/} ;; + esac + local color=auto + if [[ -t 1 ]] || [[ -n "${GITHUB_ACTIONS:-}" ]]; then + color=always + fi + if ! shellcheck --color="${color}" --exclude "${shellcheck_exclude}" <(printf '%s\n' "${text}") | sed "s/\/dev\/fd\/[0-9][0-9]*/$(sed_rhs_escape "${display_path}")/g"; then + error "check failed; please resolve the above shellcheck error(s)" + fi + } + for dockerfile_path in ${docker_files[@]+"${docker_files[@]}"}; do + dockerfile=$(parse-dockerfile "${dockerfile_path}") + normal_shell='' + for instruction in $(jq -c '.instructions[]' <<<"${dockerfile}"); do + instruction_kind=$(jq -r '.kind' <<<"${instruction}") + case "${instruction_kind}" in + FROM) + # https://docs.docker.com/reference/dockerfile/#from + # > Each FROM instruction clears any state created by previous instructions. + normal_shell='' + continue + ;; + ADD | ARG | CMD | COPY | ENTRYPOINT | ENV | EXPOSE | HEALTHCHECK | LABEL) ;; + # https://docs.docker.com/reference/build-checks/maintainer-deprecated/ + MAINTAINER) error "MAINTAINER instruction is deprecated in favor of using label" ;; + RUN) ;; + SHELL) + normal_shell='' + for argument in $(jq -c '.arguments[]' <<<"${instruction}"); do + value=$(jq -r '.value' <<<"${argument}") + if [[ -z "${normal_shell}" ]]; then + case "${value}" in + cmd | cmd.exe | powershell | powershell.exe) + # not unix shell + normal_shell="${value}" + break + ;; + esac + else + normal_shell+=' ' + fi + normal_shell+="${value}" + done + ;; + STOPSIGNAL | USER | VOLUME | WORKDIR) ;; + *) error "unknown instruction ${instruction_kind}" ;; + esac + arguments='' + # only shell-form RUN/ENTRYPOINT/CMD is run in a shell + case "${instruction_kind}" in + RUN) + if [[ "$(jq -r '.arguments.shell' <<<"${instruction}")" == 'null' ]]; then + continue + fi + arguments=$(jq -r '.arguments.shell.value' <<<"${instruction}") + if [[ -z "${arguments}" ]]; then + if [[ "$(jq -r '.here_docs[0]' <<<"${instruction}")" == 'null' ]]; then + error "empty RUN is useless (${dockerfile_path})" + continue + fi + if [[ "$(jq -r '.here_docs[1]' <<<"${instruction}")" != 'null' ]]; then + # TODO: + error "multi here-docs without command is not yet supported (${dockerfile_path})" + fi + arguments=$(jq -r '.here_docs[0].value' <<<"${instruction}") + if [[ "${arguments}" == '#!'* ]]; then + # TODO: + error "here-docs with shebang is not yet supported (${dockerfile_path})" + continue + fi + else + if [[ "$(jq -r '.here_docs[0]' <<<"${instruction}")" != 'null' ]]; then + # TODO: + error "sh/bash command with here-docs is not yet checked (${dockerfile_path})" + fi + fi + ;; + ENTRYPOINT | CMD) + if [[ "$(jq -r '.arguments.shell' <<<"${instruction}")" == 'null' ]]; then + continue + fi + arguments=$(jq -r '.arguments.shell.value' <<<"${instruction}") + if [[ -z "${normal_shell}" ]] && [[ -n "${arguments}" ]]; then + # https://docs.docker.com/reference/build-checks/json-args-recommended/ + error "JSON arguments recommended for ENTRYPOINT/CMD to prevent unintended behavior related to OS signals" + fi + ;; + HEALTHCHECK) + if [[ "$(jq -r '.arguments.kind' <<<"${instruction}")" != "CMD" ]]; then + continue + fi + if [[ "$(jq -r '.arguments.arguments.shell' <<<"${instruction}")" == 'null' ]]; then + continue + fi + arguments=$(jq -r '.arguments.arguments.shell.value' <<<"${instruction}") + ;; + *) continue ;; + esac + case "${normal_shell}" in + # not unix shell + cmd | cmd.exe | powershell | powershell.exe) continue ;; + # https://docs.docker.com/reference/dockerfile/#shell + '') shell='/bin/sh -c' ;; + *) shell="${normal_shell}" ;; + esac + shellcheck_for_dockerfile "${arguments}" "${shell}" "${dockerfile_path}" + done + done + fi + fi + # Check scripts in YAML. + if [[ ${#workflows[@]} -gt 0 ]] || [[ ${#actions[@]} -gt 0 ]]; then + # Exclude SC2096 due to the way the temporary script is created. + shellcheck_exclude=SC2086,SC2096,SC2129 + info "running \`shellcheck --exclude ${shellcheck_exclude}\` for scripts in .github/workflows/*.yml and **/action.yml" + if [[ "${ostype}" == 'windows' ]]; then + # No such file or directory: '/proc/N/fd/N' + warn "this check is skipped on Windows due to upstream bug (failed to found fd created by <())" + elif [[ "${ostype}" == 'dragonfly' ]]; then + warn "this check is skipped on DragonFly BSD due to upstream bug (hang)" + elif check_install jq python3; then + shellcheck_for_gha() { + local text=$1 + local shell=$2 + local display_path=$3 + if [[ "${text}" == 'null' ]]; then + return + fi + case "${shell}" in + bash* | sh*) ;; + *) return ;; + esac + # Use python because sed doesn't support .*?. + text=$( + "python${py_suffix}" - <(printf '%s\n%s' "#!/usr/bin/env ${shell%' {0}'}" "${text}") </dev/null; then + warn "this check is skipped on Solaris due to no node 18+ in upstream package manager" + elif [[ "${ostype}" == 'illumos' ]]; then + warn "this check is skipped on illumos due to upstream bug (dictionaries are not loaded correctly)" + elif check_install npm jq python3; then has_rust='' - if [[ -n "$(git ls-files '*Cargo.toml')" ]]; then - venv_install_yq - has_rust='1' + if [[ -n "$(ls_files '*Cargo.toml')" ]]; then + has_rust=1 dependencies='' - for manifest_path in $(git ls-files '*Cargo.toml'); do - if [[ "${manifest_path}" != "Cargo.toml" ]] && [[ "$(venv tomlq -c '.workspace' "${manifest_path}")" == "null" ]]; then + for manifest_path in $(ls_files '*Cargo.toml'); do + if [[ "${manifest_path}" != "Cargo.toml" ]] && [[ "$(tomlq -c '.workspace' "${manifest_path}")" == 'null' ]]; then continue fi - metadata=$(cargo metadata --format-version=1 --no-deps --manifest-path "${manifest_path}") - for id in $(jq <<<"${metadata}" '.workspace_members[]'); do - dependencies+="$(jq <<<"${metadata}" ".packages[] | select(.id == ${id})" | jq -r '.dependencies[].name')"$'\n' - done + m=$(cargo metadata --format-version=1 --no-deps --manifest-path "${manifest_path}" || true) + if [[ -z "${m}" ]]; then + continue # Ignore broken manifest + fi + dependencies+="$(jq -r '. as $metadata | .workspace_members[] as $id | $metadata.packages[] | select(.id == $id) | .dependencies[].name' <<<"${m}")"$'\n' done - # shellcheck disable=SC2001 - dependencies=$(sed <<<"${dependencies}" 's/[0-9_-]/\n/g' | LC_ALL=C sort -f -u) + dependencies=$(LC_ALL=C sort -f -u <<<"${dependencies//[0-9_-]/$'\n'}") fi config_old=$(<.cspell.json) - config_new=$(grep <<<"${config_old}" -v '^ *//' | jq 'del(.dictionaries[] | select(index("organization-dictionary") | not))' | jq 'del(.dictionaryDefinitions[] | select(.name == "organization-dictionary" | not))') - trap -- 'echo "${config_old}" >.cspell.json; echo >&2 "$0: trapped SIGINT"; exit 1' SIGINT - echo "${config_new}" >.cspell.json + config_new=$({ grep -Ev '^ *//' <<<"${config_old}" || true; } | jq 'del(.dictionaries[] | select(index("organization-dictionary") | not)) | del(.dictionaryDefinitions[] | select(.name == "organization-dictionary" | not))') + trap -- 'printf "%s\n" "${config_old}" >|.cspell.json; printf >&2 "%s\n" "${0##*/}: trapped SIGINT"; exit 1' SIGINT + printf '%s\n' "${config_new}" >|.cspell.json + dependencies_words='' if [[ -n "${has_rust}" ]]; then - dependencies_words=$(npx <<<"${dependencies}" -y cspell stdin --no-progress --no-summary --words-only --unique || true) + dependencies_words=$(npx -y cspell stdin --no-progress --no-summary --words-only --unique <<<"${dependencies}" || true) fi - all_words=$(npx -y cspell --no-progress --no-summary --words-only --unique $(git ls-files | (grep -v "${project_dictionary//\./\\.}" || true)) || true) - echo "${config_old}" >.cspell.json - trap - SIGINT - cat >.github/.cspell/rust-dependencies.txt <|.cspell.json + trap -- 'printf >&2 "%s\n" "${0##*/}: trapped SIGINT"; exit 1' SIGINT + cat >|.github/.cspell/rust-dependencies.txt <>.github/.cspell/rust-dependencies.txt + if [[ -n "${dependencies_words}" ]]; then + LC_ALL=C sort -f >>.github/.cspell/rust-dependencies.txt <<<"${dependencies_words}"$'\n' + fi + if [[ -z "${CI:-}" ]]; then + REMOVE_UNUSED_WORDS=1 fi if [[ -z "${REMOVE_UNUSED_WORDS:-}" ]]; then check_diff .github/.cspell/rust-dependencies.txt @@ -434,9 +1014,12 @@ EOF error "you may want to mark .github/.cspell/rust-dependencies.txt linguist-generated" fi - info "running \`npx -y cspell --no-progress --no-summary \$(git ls-files)\`" - if ! npx -y cspell --no-progress --no-summary $(git ls-files); then - error "spellcheck failed: please fix uses of above words or add to ${project_dictionary} if correct" + info "running \`git ls-files | npx -y cspell --file-list stdin --no-progress --no-summary\`" + if ! ls_files | npx -y cspell --file-list stdin --no-progress --no-summary; then + error "spellcheck failed: please fix uses of below words or add to ${project_dictionary} if correct" + printf '=======================================\n' + { ls_files | npx -y cspell --file-list stdin --no-progress --no-summary --words-only || true; } | sed "s/'s$//g" | LC_ALL=C sort -f -u + printf '=======================================\n\n' fi # Make sure the project-specific dictionary does not contain duplicated words. @@ -444,43 +1027,48 @@ EOF if [[ "${dictionary}" == "${project_dictionary}" ]]; then continue fi - dup=$(sed '/^$/d' "${project_dictionary}" "${dictionary}" | LC_ALL=C sort -f | uniq -d -i | (grep -v '//.*' || true)) + case "${ostype}" in + # NetBSD uniq doesn't support -i flag. + netbsd) dup=$(sed '/^$/d; /^\/\//d' "${project_dictionary}" "${dictionary}" | LC_ALL=C sort -f | tr '[:upper:]' '[:lower:]' | LC_ALL=C uniq -d) ;; + *) dup=$(sed '/^$/d; /^\/\//d' "${project_dictionary}" "${dictionary}" | LC_ALL=C sort -f | LC_ALL=C uniq -d -i) ;; + esac if [[ -n "${dup}" ]]; then error "duplicated words in dictionaries; please remove the following words from ${project_dictionary}" - echo "=======================================" - echo "${dup}" - echo "=======================================" + print_fenced "${dup}"$'\n' fi done # Make sure the project-specific dictionary does not contain unused words. if [[ -n "${REMOVE_UNUSED_WORDS:-}" ]]; then grep_args=() - for word in $(grep -Ev '^//.*' "${project_dictionary}" || true); do + for word in $(grep -Ev '^//' "${project_dictionary}" || true); do if ! grep -Eqi "^${word}$" <<<"${all_words}"; then grep_args+=(-e "^${word}$") fi done if [[ ${#grep_args[@]} -gt 0 ]]; then info "removing unused words from ${project_dictionary}" - res=$(grep -Ev "${grep_args[@]}" "${project_dictionary}") - printf '%s\n' "${res}" >|"${project_dictionary}" + res=$(grep -Ev "${grep_args[@]}" "${project_dictionary}" || true) + if [[ -n "${res}" ]]; then + printf '%s\n' "${res}" >|"${project_dictionary}" + else + printf '' >|"${project_dictionary}" + fi fi else unused='' - for word in $(grep -Ev '^//.*' "${project_dictionary}" || true); do + for word in $(grep -Ev '^//' "${project_dictionary}" || true); do if ! grep -Eqi "^${word}$" <<<"${all_words}"; then unused+="${word}"$'\n' fi done if [[ -n "${unused}" ]]; then - error "unused words in dictionaries; please remove the following words from ${project_dictionary} or run ${0##*/} with REMOVE_UNUSED_WORDS=1" - printf '=======================================\n' - printf '%s' "${unused}" - printf '=======================================\n' + error "unused words in dictionaries; please remove the following words from ${project_dictionary} or run ${0##*/} locally" + print_fenced "${unused}" fi fi fi + printf '\n' fi if [[ -n "${should_fail:-}" ]]; then