Skip to content

Commit

Permalink
reviewdog: move scripts in cmd/ directory
Browse files Browse the repository at this point in the history
  • Loading branch information
thypon committed Mar 20, 2024
1 parent d14e3ba commit acf35cc
Show file tree
Hide file tree
Showing 10 changed files with 154 additions and 75 deletions.
2 changes: 1 addition & 1 deletion assets/reviewdog.sh
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ export PATH=$PATH:$GOROOT/bin:$GOPATH/bin
export SEC_ACTION_DEBUG=$SEC_ACTION_DEBUG
export ASSIGNEES=$(echo "$ASSIGNEES" | sed 's|\([^ ]\)|@\1|' | tr -s '\n' ' ')

RUNNERS="safesvg tfsec semgrep sveltegrep npm-audit pip-audit" # disabled: brakeman
RUNNERS="tfsec semgrep savesvg sveltegrep npm-audit pip-audit" # disabled: brakeman

if [ -n "${GITHUB_BASE_REF+set}" ]; then
for runner in $RUNNERS; do
Expand Down
24 changes: 24 additions & 0 deletions assets/reviewdog/cmd/brakeman.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
#!/bin/sh
set -e
if command -v gxargs > /dev/null; then
alias xargs=gxargs
fi
(if xargs -0 -n1 -a $SCRIPTPATH/all_changed_files.txt | grep -q '\.rb$'; then
bundle exec brakeman --quiet \
--no-exit-on-warn \
--no-exit-on-error \
--skip-files vendor/ \
--skip-libs \
--force \
--format json |
jq -r '.warnings[] | "\(.confidence[0:1]):\(.file):\(.line) \(.message | sub("\n";"<br/>";"g"))<br><br>Source: \(.link)"' |
$SCRIPTPATH/cleaner.rb
fi) 2>reviewdog.brakeman.stderr.log >reviewdog.brakeman.log

# check if we are running in REVIEWDOG_MODE, if not print stderr and output, otherwise only stout
if [ -z "$REVIEWDOG_MODE" ]; then
cat reviewdog.brakeman.stderr.log
cat reviewdog.brakeman.log
else
cat reviewdog.brakeman.log
fi
15 changes: 15 additions & 0 deletions assets/reviewdog/cmd/npm-audit.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
#!/bin/sh
set -e
if command -v gxargs > /dev/null; then
alias xargs=gxargs
fi
(python3 $SCRIPTPATH/npm-audit.py |
$SCRIPTPATH/cleaner.rb) 2>reviewdog.npm-audit.stderr.log >reviewdog.npm-audit.log

# check if we are running in REVIEWDOG_MODE, if not print stderr and output, otherwise only stout
if [ -z "$REVIEWDOG_MODE" ]; then
cat reviewdog.npm-audit.stderr.log
cat reviewdog.npm-audit.log
else
cat reviewdog.npm-audit.log
fi
15 changes: 15 additions & 0 deletions assets/reviewdog/cmd/pip-audit.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
#!/bin/sh
set -e
if command -v gxargs > /dev/null; then
alias xargs=gxargs
fi
(python3 $SCRIPTPATH/pip-audit.py |
$SCRIPTPATH/cleaner.rb) 2>reviewdog.pip-audit.stderr.log >reviewdog.pip-audit.log

# check if we are running in REVIEWDOG_MODE, if not print stderr and output, otherwise only stout
if [ -z "$REVIEWDOG_MODE" ]; then
cat reviewdog.pip-audit.stderr.log
cat reviewdog.pip-audit.log
else
cat reviewdog.pip-audit.log
fi
16 changes: 16 additions & 0 deletions assets/reviewdog/cmd/safesvg.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
#!/bin/sh
set -e
# if gxargs is defined define xargs alias
if command -v gxargs > /dev/null; then
alias xargs=gxargs
fi
(xargs -0 -n1 -a $SCRIPTPATH/all_changed_files.txt $SCRIPTPATH/xmllint.sh |
$SCRIPTPATH/cleaner.rb --svgo) 2>reviewdog.safesvg.stderr.log >reviewdog.safesvg.log

# check if we are running in REVIEWDOG_MODE, if not print stderr and output, otherwise only stout
if [ -z "$REVIEWDOG_MODE" ]; then
cat reviewdog.safesvg.stderr.log
cat reviewdog.safesvg.log
else
cat reviewdog.safesvg.log
fi
22 changes: 22 additions & 0 deletions assets/reviewdog/cmd/semgrep.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
#!/bin/sh
set -e
(semgrep \
-c $SCRIPTPATH/semgrep_rules/generated/oss/vulns.yaml \
-c $SCRIPTPATH/semgrep_rules/generated/nonfree/vulns.yaml \
-c $SCRIPTPATH/semgrep_rules/generated/oss/audit.yaml \
-c $SCRIPTPATH/semgrep_rules/generated/nonfree/audit.yaml \
$(find $SCRIPTPATH/semgrep_rules -name '*.yml' -or -name '*.yaml' -not -name "*.test.yml" -not -name "*.test.yaml" -not -path "$SCRIPTPATH/semgrep_rules/generated/*" | sed 's/^/-c /g') \
--metrics=off \
--quiet \
$([ -n "${GITHUB_BASE_REF+set}" ] && echo "--baseline-commit origin/${GITHUB_BASE_REF:-main}") \
--json |
jq -r '.results[] | "\(.extra.severity[0:1]):\(.path):\(.end.line) \(.extra.message | sub("\n";"<br/>";"g"))<br><br>Source: \(.extra.metadata.source)<br><br>,\(if .extra.metadata.assignees then .extra.metadata.assignees else "null" end | sub("\n";" ";"g"))"' |
$SCRIPTPATH/cleaner.rb --semgrep --assignees) 2>reviewdog.semgrep.stderr.log >reviewdog.semgrep.log

# check if we are running in REVIEWDOG_MODE, if not print stderr and output, otherwise only stout
if [ -z "$REVIEWDOG_MODE" ]; then
cat reviewdog.semgrep.stderr.log
cat reviewdog.semgrep.log
else
cat reviewdog.semgrep.log
fi
36 changes: 36 additions & 0 deletions assets/reviewdog/cmd/sveltegrep.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
#!/bin/sh
set -e
(python3 $SCRIPTPATH/scripttagextractor.py \
--suffix .extractedscript.js \
--ignore-no-files \
--all-changed-files-suffix .html &&
python3 $SCRIPTPATH/scripttagextractor.py \
--add-suffix-to-original .extractedscript.html \
--suffix .extractedscript.ts \
--ignore-no-files \
--all-changed-files-suffix .svelte &&
semgrep \
-c $SCRIPTPATH/semgrep_rules/generated/oss/vulns.yaml \
-c $SCRIPTPATH/semgrep_rules/generated/nonfree/vulns.yaml \
-c $SCRIPTPATH/semgrep_rules/generated/oss/audit.yaml \
-c $SCRIPTPATH/semgrep_rules/generated/nonfree/audit.yaml \
$(find $SCRIPTPATH/semgrep_rules -name '*.yml' -or -name '*.yaml' -not -name "*.test.yml" -not -name "*.test.yaml" -not -path "$SCRIPTPATH/semgrep_rules/generated/*" | sed 's/^/-c /g') \
--metrics=off \
--json \
--quiet \
--no-git-ignore \
'--include=*.extractedscript.ts' \
'--include=*.extractedscript.js' \
'--include=*.extractedscript.html' \
./ |
jq -r '.results[] | "\(.extra.severity[0:1]):\(.path):\(.end.line) \(.extra.message | sub("\n";"<br/>";"g"))<br><br>Source: \(.extra.metadata.source)<br><br>,\(if .extra.metadata.assignees then .extra.metadata.assignees else "null" end | sub("\n";" ";"g"))"' |
$SCRIPTPATH/cleaner.rb --assignees --sveltegrep &&
find . -type f -name '*.extractedscript.*' -delete) 2>reviewdog.sveltegrep.stderr.log >reviewdog.sveltegrep.log

# check if we are running in REVIEWDOG_MODE, if not print stderr and output, otherwise only stout
if [ -z "$REVIEWDOG_MODE" ]; then
cat reviewdog.sveltegrep.stderr.log
cat reviewdog.sveltegrep.log
else
cat reviewdog.sveltegrep.log
fi
17 changes: 17 additions & 0 deletions assets/reviewdog/cmd/tfsec.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
#!/bin/sh
set -e
if command -v gxargs > /dev/null; then
alias xargs=gxargs
fi
(xargs -0 -n1 -a $SCRIPTPATH/all_changed_files.txt | grep '\.tf$' | xargs -r -d '\n' dirname | sort -u |
xargs -r -d '\n' $SCRIPTPATH/tfsec.sh |
jq -r '.diagnostics[] | "\(.severity[0:1]):\(.location.path):\(.location.range.start.line) \(.message | sub("\n";"<br/>";"g"))<br><br>source: \(.code.url)<br><br>"' |
$SCRIPTPATH/cleaner.rb) 2>reviewdog.tfsec.stderr.log >reviewdog.tfsec.log

# check if we are running in REVIEWDOG_MODE, if not print stderr and output, otherwise only stout
if [ -z "$REVIEWDOG_MODE" ]; then
cat reviewdog.tfsec.stderr.log
cat reviewdog.tfsec.log
else
cat reviewdog.tfsec.log
fi
80 changes: 7 additions & 73 deletions assets/reviewdog/reviewdog.yml
Original file line number Diff line number Diff line change
@@ -1,102 +1,36 @@
runner:
semgrep:
name: semgrep
cmd: |
set -e
(semgrep \
-c $SCRIPTPATH/semgrep_rules/generated/oss/vulns.yaml \
-c $SCRIPTPATH/semgrep_rules/generated/nonfree/vulns.yaml \
-c $SCRIPTPATH/semgrep_rules/generated/oss/audit.yaml \
-c $SCRIPTPATH/semgrep_rules/generated/nonfree/audit.yaml \
$(find $SCRIPTPATH/semgrep_rules -name '*.yml' -or -name '*.yaml' -not -name "*.test.yml" -not -name "*.test.yaml" -not -path "$SCRIPTPATH/semgrep_rules/generated/*" | sed 's/^/-c /g') \
--metrics=off \
--quiet \
$([ -n "${GITHUB_BASE_REF+set}" ] && echo "--baseline-commit origin/${GITHUB_BASE_REF:-main}") \
--json \
| jq -r '.results[] | "\(.extra.severity[0:1]):\(.path):\(.end.line) \(.extra.message | sub("\n";"<br/>";"g"))<br><br>Source: \(.extra.metadata.source)<br><br>,\(if .extra.metadata.assignees then .extra.metadata.assignees else "null" end | sub("\n";" ";"g"))"' \
| $SCRIPTPATH/cleaner.rb --semgrep --assignees) 2> reviewdog.semgrep.stderr.log
cmd: REVIEWDOG_MODE=1 $SCRIPTPATH/reviewdog/cmd/semgrep.sh
errorformat:
- "%t:%f:%l %m"
sveltegrep:
name: sveltegrep
cmd: |
set -e
(python3 $SCRIPTPATH/scripttagextractor.py \
--suffix .extractedscript.js \
--ignore-no-files \
--all-changed-files-suffix .html && \
python3 $SCRIPTPATH/scripttagextractor.py \
--add-suffix-to-original .extractedscript.html \
--suffix .extractedscript.ts \
--ignore-no-files \
--all-changed-files-suffix .svelte && \
semgrep \
-c $SCRIPTPATH/semgrep_rules/generated/oss/vulns.yaml \
-c $SCRIPTPATH/semgrep_rules/generated/nonfree/vulns.yaml \
-c $SCRIPTPATH/semgrep_rules/generated/oss/audit.yaml \
-c $SCRIPTPATH/semgrep_rules/generated/nonfree/audit.yaml \
$(find $SCRIPTPATH/semgrep_rules -name '*.yml' -or -name '*.yaml' -not -name "*.test.yml" -not -name "*.test.yaml" -not -path "$SCRIPTPATH/semgrep_rules/generated/*" | sed 's/^/-c /g') \
--metrics=off \
--json \
--quiet \
--no-git-ignore \
'--include=*.extractedscript.ts' \
'--include=*.extractedscript.js' \
'--include=*.extractedscript.html' \
./ \
| jq -r '.results[] | "\(.extra.severity[0:1]):\(.path):\(.end.line) \(.extra.message | sub("\n";"<br/>";"g"))<br><br>Source: \(.extra.metadata.source)<br><br>,\(if .extra.metadata.assignees then .extra.metadata.assignees else "null" end | sub("\n";" ";"g"))"' \
| $SCRIPTPATH/cleaner.rb --assignees --sveltegrep && \
find . -type f -name '*.extractedscript.*' -delete) 2> reviewdog.sveltegrep.stderr.log
cmd: REVIEWDOG_MODE=1 $SCRIPTPATH/reviewdog/cmd/sveltegrep.sh
errorformat:
- "%t:%f:%l %m"
safesvg:
name: safesvg
cmd: |
set -e
(xargs -0 -n1 -a $SCRIPTPATH/all_changed_files.txt $SCRIPTPATH/xmllint.sh \
| $SCRIPTPATH/cleaner.rb --svgo) 2> reviewdog.safesvg.stderr.log
cmd: REVIEWDOG_MODE=1 $SCRIPTPATH/reviewdog/cmd/savesvg.sh
errorformat:
- "%f:%l: %m"
tfsec:
name: tfsec
cmd: |
set -e
(xargs -0 -n1 -a $SCRIPTPATH/all_changed_files.txt | grep '\.tf$' | xargs -r -d '\n' dirname | sort -u \
| xargs -r -d '\n' $SCRIPTPATH/tfsec.sh \
| jq -r '.diagnostics[] | "\(.severity[0:1]):\(.location.path):\(.location.range.start.line) \(.message | sub("\n";"<br/>";"g"))<br><br>source: \(.code.url)<br><br>"' \
| $SCRIPTPATH/cleaner.rb) 2> reviewdog.tfsec.stderr.log
cmd: REVIEWDOG_MODE=1 $SCRIPTPATH/reviewdog/cmd/tfsec.sh
errorformat:
- "%t:%f:%l %m"
brakeman:
name: brakeman
cmd: |
set -e
(if xargs -0 -n1 -a $SCRIPTPATH/all_changed_files.txt | grep -q '\.rb$'; then
bundle exec brakeman --quiet \
--no-exit-on-warn \
--no-exit-on-error \
--skip-files vendor/ \
--skip-libs \
--force \
--format json \
| jq -r '.warnings[] | "\(.confidence[0:1]):\(.file):\(.line) \(.message | sub("\n";"<br/>";"g"))<br><br>Source: \(.link)"' \
| $SCRIPTPATH/cleaner.rb
fi) 2> reviewdog.brakeman.stderr.log
cmd: REVIEWDOG_MODE=1 $SCRIPTPATH/reviewdog/cmd/brakeman.sh
errorformat:
- "%t:%f:%l %m"
npm-audit:
name: npm-audit
cmd: |
set -e
(python3 $SCRIPTPATH/npm-audit.py \
| $SCRIPTPATH/cleaner.rb) 2> reviewdog.npm-audit.stderr.log
cmd: REVIEWDOG_MODE=1 $SCRIPTPATH/reviewdog/cmd/npm-audit.sh
errorformat:
- "%t:%f:%l %m"
pip-audit:
name: pip-audit
cmd: |
set -e
(python3 $SCRIPTPATH/pip-audit.py \
| $SCRIPTPATH/cleaner.rb) 2> reviewdog.pip-audit.stderr.log
cmd: REVIEWDOG_MODE=1 $SCRIPTPATH/reviewdog/cmd/pip-audit.sh
errorformat:
- "%t:%f:%l %m"
2 changes: 1 addition & 1 deletion assets/xmllint.sh
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
# Absolute path to this script. /home/user/bin/foo.sh
SCRIPT=$(readlink -f $0)
# Absolute path this script is in. /home/user/bin
export SCRIPTPATH=`dirname $SCRIPT`
SCRIPTPATH=`dirname $SCRIPT`

# Only check SVGs
[[ "$1" == *".svg" ]] || exit 0
Expand Down

0 comments on commit acf35cc

Please sign in to comment.