diff --git a/.github/.cspell/rust-dependencies.txt b/.github/.cspell/rust-dependencies.txt index f41ad51b..6e738206 100644 --- a/.github/.cspell/rust-dependencies.txt +++ b/.github/.cspell/rust-dependencies.txt @@ -1,5 +1,5 @@ -// This file is @generated by tidy.sh. -// It is not intended for manual editing. +# This file is @generated by tidy.sh. +# It is not intended for manual editing. flate minisign diff --git a/.github/zizmor.yml b/.github/zizmor.yml index 789f8c87..e97a3c9a 100644 --- a/.github/zizmor.yml +++ b/.github/zizmor.yml @@ -2,7 +2,7 @@ # https://docs.zizmor.sh/configuration/ rules: - dependabot-cooldown: { disable: true } # Useless unless unpinned-uses is enabled. + dependabot-cooldown: { disable: true } # Useless unless hash-pin is forced by unpinned-uses. ref-confusion: { disable: true } # TODO: Old GHA didn't work without this pattern in some cases, but does it seem to be fixed? secrets-inherit: { disable: true } unpinned-uses: @@ -12,5 +12,5 @@ rules: '*': ref-pin obfuscation: ignore: - # We use `shell: cmd` to test compatibility + # We use `shell: cmd` to test compatibility. - ci.yml diff --git a/tools/publish.sh b/tools/publish.sh index 5e3f18cc..c3b96134 100755 --- a/tools/publish.sh +++ b/tools/publish.sh @@ -191,7 +191,7 @@ cp -- ./manifests/* "${schema_workspace}" # Stage changes git add . # Detect changes, then commit and push if changes exist - if [[ "$(git status --porcelain=v1 | wc -l)" != "0" ]]; then + if [[ "$(git status --porcelain=v1 | LC_ALL=C wc -l)" != "0" ]]; then git commit -m 'Update manifest schema' retry git push origin HEAD fi diff --git a/tools/tidy.sh b/tools/tidy.sh index a877974e..ceec4c7b 100755 --- a/tools/tidy.sh +++ b/tools/tidy.sh @@ -118,11 +118,11 @@ check_alt() { check_hidden() { local res for file in "$@"; do - check_alt ".${file}" "${file}" "$(comm -23 <(ls_files "*${file}") <(ls_files "*.${file}"))" + check_alt ".${file}" "${file}" "$(LC_ALL=C comm -23 <(ls_files "*${file}") <(ls_files "*.${file}"))" done } sed_rhs_escape() { - sed 's/\\/\\\\/g; s/\&/\\\&/g; s/\//\\\//g' <<<"$1" + sed -E 's/\\/\\\\/g; s/\&/\\\&/g; s/\//\\\//g' <<<"$1" } if [[ $# -gt 0 ]]; then @@ -137,12 +137,8 @@ py_suffix='' if type -P python3 >/dev/null; then py_suffix=3 fi -yq() { - pipx run yq "$@" -} -tomlq() { - pipx run --spec yq tomlq "$@" -} +yq() { pipx run yq "$@"; } +tomlq() { pipx run --spec yq tomlq "$@"; } case "$(uname -s)" in Linux) if [[ "$(uname -o)" == 'Android' ]]; then @@ -167,10 +163,11 @@ case "$(uname -s)" in if [[ "${PATH}" != *'/usr/xpg4/bin'* ]]; then export PATH="/usr/xpg4/bin:${PATH}" fi - # GNU/BSD grep/sed is required to run some checks, but most checks are okay with other POSIX grep/sed. + # GNU/BSD sed is required. + # GNU/BSD grep is required by some checks, but most checks are okay with other POSIX grep. # Solaris /usr/xpg4/bin/grep has -q, -E, -F, but no -o (non-POSIX). # Solaris /usr/xpg4/bin/sed has no -E (POSIX.1-2024) yet. - for tool in sed grep; do + for tool in 'grep' 'sed'; do if type -P "g${tool}" >/dev/null; then eval "${tool}() { g${tool} \"\$@\"; }" fi @@ -189,12 +186,8 @@ case "$(uname -s)" in else jq() { command jq "$@" | tr -d '\r'; } fi - yq() { - pipx run yq "$@" | tr -d '\r' - } - tomlq() { - pipx run --spec yq tomlq "$@" | tr -d '\r' - } + yq() { pipx run yq "$@" | tr -d '\r'; } + tomlq() { pipx run --spec yq tomlq "$@" | tr -d '\r'; } fi fi ;; @@ -203,25 +196,34 @@ esac check_install git exclude_from_ls_files=() -# - `find` lists symlinks. `! ( -name -prune )` (.i.e., ignore ) are manually listed from .gitignore. -# - `git submodule status` lists submodules. Use sed to remove the first character indicates status ( |+|-). +# - `find` lists symlinks. `! ( -name -prune )` means recursively ignore . `cut` removes the leading `./`. +# This can be replaced with `fd -H -t l`. +# - `git submodule status` lists submodules. The first `cut` removes the first character indicates status ( |+|-). # - `git ls-files --deleted` lists removed files. -while IFS=$'\n' read -r line; do exclude_from_ls_files+=("${line}"); done < <({ - find . \! \( -name .git -prune \) \! \( -name target -prune \) \! \( -name tmp -prune \) -type l | cut -c3- - git submodule status | sed 's/^.//' | cut -d' ' -f2 +find_prune=(\! \( -name .git -prune \)) +while IFS= read -r; do + find_prune+=(\! \( -name "${REPLY}" -prune \)) +done < <(sed -E 's/#.*//g; s/^[ \t]+//g; s/\/[ \t]+$//g; /^$/d' .gitignore) +while IFS=$'\n' read -r; do + exclude_from_ls_files+=("${REPLY}") +done < <({ + find . "${find_prune[@]}" -type l | cut -c3- + git submodule status | cut -c2- | cut -d' ' -f2 git ls-files --deleted } | LC_ALL=C sort -u) exclude_from_ls_files_no_symlink=() -while IFS=$'\n' read -r line; do exclude_from_ls_files_no_symlink+=("${line}"); done < <({ - git submodule status | sed 's/^.//' | cut -d' ' -f2 +while IFS=$'\n' read -r; do + exclude_from_ls_files_no_symlink+=("${REPLY}") +done < <({ + git submodule status | cut -c2- | cut -d' ' -f2 git ls-files --deleted } | LC_ALL=C sort -u) ls_files() { if [[ "${1:-}" == '--include-symlink' ]]; then shift - comm -23 <(git ls-files "$@" | LC_ALL=C sort) <(printf '%s\n' ${exclude_from_ls_files_no_symlink[@]+"${exclude_from_ls_files_no_symlink[@]}"}) + LC_ALL=C comm -23 <(git ls-files "$@" | LC_ALL=C sort) <(printf '%s\n' ${exclude_from_ls_files_no_symlink[@]+"${exclude_from_ls_files_no_symlink[@]}"}) else - comm -23 <(git ls-files "$@" | LC_ALL=C sort) <(printf '%s\n' ${exclude_from_ls_files[@]+"${exclude_from_ls_files[@]}"}) + LC_ALL=C comm -23 <(git ls-files "$@" | LC_ALL=C sort) <(printf '%s\n' ${exclude_from_ls_files[@]+"${exclude_from_ls_files[@]}"}) fi } @@ -445,7 +447,7 @@ if [[ -n "$(ls_files '*.rs')" ]]; then new+="${line}"$'\a' done < <(tr '\n' '\a' <"${markdown}" | grep -Eo '.*') new+='' - new=$(tr '\n' '\a' <"${lib}" | sed "s/.*/$(sed_rhs_escape "${new}")/" | tr '\a' '\n') + new=$(tr '\n' '\a' <"${lib}" | sed -E "s/.*/$(sed_rhs_escape "${new}")/" | tr '\a' '\n') printf '%s\n' "${new}" >|"${lib}" check_diff "${lib}" done @@ -677,7 +679,7 @@ elif check_install shellcheck; then # Others: false negative trap -- 'rm -- ./tools/.tidy-tmp; printf >&2 "%s\n" "${0##*/}: trapped SIGINT"; exit 1' SIGINT printf '%s\n' "${text}" >|./tools/.tidy-tmp - if ! shellcheck --color="${color}" --exclude "${shellcheck_exclude}" ./tools/.tidy-tmp | sed "s/\.\/tools\/\.tidy-tmp/$(sed_rhs_escape "${display_path}")/g"; then + if ! shellcheck --color="${color}" --exclude "${shellcheck_exclude}" ./tools/.tidy-tmp | sed -E "s/\.\/tools\/\.tidy-tmp/$(sed_rhs_escape "${display_path}")/g"; then error "check failed; please resolve the above shellcheck error(s)" fi rm -- ./tools/.tidy-tmp @@ -822,7 +824,7 @@ EOF # Others: false negative trap -- 'rm -- ./tools/.tidy-tmp; printf >&2 "%s\n" "${0##*/}: trapped SIGINT"; exit 1' SIGINT printf '%s\n' "${text}" >|./tools/.tidy-tmp - if ! shellcheck --color="${color}" --exclude "${shellcheck_exclude}" ./tools/.tidy-tmp | sed "s/\.\/tools\/\.tidy-tmp/$(sed_rhs_escape "${display_path}")/g"; then + if ! shellcheck --color="${color}" --exclude "${shellcheck_exclude}" ./tools/.tidy-tmp | sed -E "s/\.\/tools\/\.tidy-tmp/$(sed_rhs_escape "${display_path}")/g"; then error "check failed; please resolve the above shellcheck error(s)" fi rm -- ./tools/.tidy-tmp @@ -833,7 +835,8 @@ EOF # The top-level permissions must be weak as they are referenced by all jobs. permissions=$(jq -c '.permissions' <<<"${workflow}") case "${permissions}" in - '{"contents":"read"}' | '{"contents":"none"}') ;; + # `permissions: {}` means "all none": https://docs.github.com/en/actions/reference/workflows-and-actions/workflow-syntax#defining-access-for-the-github_token-scopes + '{"contents":"read"}' | '{}') ;; null) error "${workflow_path}: top level permissions not found; it must be 'contents: read' or weaker permissions" ;; *) error "${workflow_path}: only 'contents: read' and weaker permissions are allowed at top level, but found '${permissions}'; if you want to use stronger permissions, please set job-level permissions" ;; esac @@ -911,11 +914,13 @@ if [[ -e .github/dependabot.yml ]]; then zizmor_targets+=(.github/dependabot.yml) fi if [[ ${#zizmor_targets[@]} -gt 0 ]]; then - if check_install zizmor; then + if [[ "${ostype}" =~ ^(netbsd|openbsd|dragonfly|illumos|solaris)$ ]] && [[ -n "${CI:-}" ]] && ! type -P zizmor >/dev/null; then + warn "this check is skipped on NetBSD/OpenBSD/Dragonfly/illumos/Solaris due to installing zizmor is hard on these platform" + elif check_install zizmor; then IFS=' ' - info "running \`zizmor ${zizmor_targets[*]}\`" + info "running \`zizmor -q ${zizmor_targets[*]}\`" IFS=$'\n\t' - zizmor "${zizmor_targets[@]}" + zizmor -q "${zizmor_targets[@]}" fi fi printf '\n' @@ -926,7 +931,7 @@ check_alt '.sh extension' '*.bash extension' "$(ls_files '*.bash')" if [[ -f tools/.tidy-check-license-headers ]]; then info "checking license headers (experimental)" failed_files='' - for p in $(comm -12 <(eval $(|.cspell.json trap -- 'printf >&2 "%s\n" "${0##*/}: trapped SIGINT"; exit 1' SIGINT cat >|.github/.cspell/rust-dependencies.txt <>.github/.cspell/rust-dependencies.txt <<<"${dependencies_words}"$'\n' @@ -1019,7 +1024,7 @@ EOF if ! ls_files | npx -y cspell stdin --no-progress --no-summary --show-context; then error "spellcheck failed: please fix uses of below words in file names or add to ${project_dictionary} if correct" printf '=======================================\n' - { ls_files | npx -y cspell stdin --no-progress --no-summary --words-only || true; } | sed "s/'s$//g" | LC_ALL=C sort -f -u + { ls_files | npx -y cspell stdin --no-progress --no-summary --words-only || true; } | sed -E "s/'s$//g" | LC_ALL=C sort -f -u printf '=======================================\n\n' fi # Check file contains. @@ -1027,7 +1032,7 @@ EOF if ! ls_files | npx -y cspell --file-list stdin --no-progress --no-summary; then error "spellcheck failed: please fix uses of below words or add to ${project_dictionary} if correct" printf '=======================================\n' - { ls_files | npx -y cspell --file-list stdin --no-progress --no-summary --words-only || true; } | sed "s/'s$//g" | LC_ALL=C sort -f -u + { ls_files | npx -y cspell --file-list stdin --no-progress --no-summary --words-only || true; } | sed -E "s/'s$//g" | LC_ALL=C sort -f -u printf '=======================================\n\n' fi @@ -1038,8 +1043,8 @@ EOF fi case "${ostype}" in # NetBSD uniq doesn't support -i flag. - netbsd) dup=$(sed '/^$/d; /^\/\//d' "${project_dictionary}" "${dictionary}" | LC_ALL=C sort -f | tr '[:upper:]' '[:lower:]' | LC_ALL=C uniq -d) ;; - *) dup=$(sed '/^$/d; /^\/\//d' "${project_dictionary}" "${dictionary}" | LC_ALL=C sort -f | LC_ALL=C uniq -d -i) ;; + netbsd) dup=$(sed -E 's/#.*//g; s/^[ \t]+//g; s/\/[ \t]+$//g; /^$/d' "${project_dictionary}" "${dictionary}" | LC_ALL=C sort -f | tr '[:upper:]' '[:lower:]' | LC_ALL=C uniq -d) ;; + *) dup=$(sed -E 's/#.*//g; s/^[ \t]+//g; s/\/[ \t]+$//g; /^$/d' "${project_dictionary}" "${dictionary}" | LC_ALL=C sort -f | LC_ALL=C uniq -d -i) ;; esac if [[ -n "${dup}" ]]; then error "duplicated words in dictionaries; please remove the following words from ${project_dictionary}" @@ -1050,13 +1055,14 @@ EOF # Make sure the project-specific dictionary does not contain unused words. if [[ -n "${REMOVE_UNUSED_WORDS:-}" ]]; then grep_args=() - for word in $(grep -Ev '^//' "${project_dictionary}" || true); do + while IFS= read -r word; do if ! grep -Eqi "^${word}$" <<<"${all_words}"; then - grep_args+=(-e "^${word}$") + grep_args+=(-e "^[ \t]*${word}[ \t]*(#.*|$)") fi - done + done < <(sed -E 's/#.*//g; s/^[ \t]+//g; s/\/[ \t]+$//g; /^$/d' "${project_dictionary}") if [[ ${#grep_args[@]} -gt 0 ]]; then info "removing unused words from ${project_dictionary}" + info "please commit changes made by the removal above" res=$(grep -Ev "${grep_args[@]}" "${project_dictionary}" || true) if [[ -n "${res}" ]]; then printf '%s\n' "${res}" >|"${project_dictionary}" @@ -1066,11 +1072,11 @@ EOF fi else unused='' - for word in $(grep -Ev '^//' "${project_dictionary}" || true); do + while IFS= read -r word; do if ! grep -Eqi "^${word}$" <<<"${all_words}"; then unused+="${word}"$'\n' fi - done + done < <(sed -E 's/#.*//g; s/^[ \t]+//g; s/\/[ \t]+$//g; /^$/d' "${project_dictionary}") if [[ -n "${unused}" ]]; then error "unused words in dictionaries; please remove the following words from ${project_dictionary} or run ${0##*/} locally" print_fenced "${unused}"