-
Notifications
You must be signed in to change notification settings - Fork 102
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
3 changed files
with
297 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,123 @@ | ||
| #!/bin/bash | ||
| if [[ "$1" == "" ]]; then | ||
| echo "must specify catalog id" | ||
| exit | ||
| fi | ||
| if [[ "$2" == "" ]]; then | ||
| echo "must specify database name" | ||
| exit | ||
| fi | ||
| if [[ "$3" == "" ]]; then | ||
| echo "must specify tablee name" | ||
| exit | ||
| fi | ||
|
|
||
| cmd[0]="$AWS glue get-partitions --catalog-id $1 --database-name $2 --table-name $3" | ||
| pref[0]="Partitions" | ||
|
|
||
| #if [[ "$4" != "" ]]; then | ||
| # cmd[0]="$AWS glue get-partition --catalog-id $1 --database-name $2 --table-name $3 --partition-values $4" | ||
| # pref[0]="Partition" | ||
| #else | ||
| # cmd[0]="$AWS glue get-partitions --catalog-id $1 --database-name $3 --table-name $3" | ||
| # pref[0]="Partitions" | ||
| #fi | ||
|
|
||
| idfilt[0]="Values" | ||
| tft[0]="aws_glue_partition" | ||
|
|
||
| #pks=$(cat $tfa.json | jq .values.partition_keys) | ||
| #pcount=`echo $pks | jq ". | length"` | ||
| #if [ "$pcount" -gt "0" ]; then | ||
| # pcount=`expr $pcount - 1` | ||
| # for i in `seq 0 $pcount`; do | ||
| # tp=`echo $pks | jq -r ".[(${i})].name"` | ||
| # echo "partition=$tp" | ||
| # ../../scripts/get-glue-partition.sh $catid $dbnam $rname $tp | ||
| # done | ||
| #fi | ||
|
|
||
|
|
||
| for c in `seq 0 0`; do | ||
|
|
||
| cm=${cmd[$c]} | ||
| ttft=${tft[(${c})]} | ||
| #echo $cm | ||
| awsout=`eval $cm 2> /dev/null` | ||
| if [ "$awsout" == "" ];then | ||
| echo "$cm : You don't have access for this resource" | ||
| exit | ||
| fi | ||
|
|
||
| count=`echo $awsout | jq ".${pref[(${c})]} | length"` | ||
| #echo "found $count partitions" | ||
|
|
||
| if [ "$count" -gt "0" ]; then | ||
| count=`expr $count - 1` | ||
| for i in `seq 0 $count`; do | ||
| #echo $i | ||
|
|
||
| cname=`echo $awsout | jq -r ".${pref[(${c})]}[(${i})].${idfilt[(${c})]}[(${c})]"` | ||
| catid=`echo $awsout | jq -r ".${pref[(${c})]}[(${i})].CatalogId"` | ||
| dbnam=`echo $awsout | jq -r ".${pref[(${c})]}[(${i})].DatabaseName"` | ||
| tbnam=`echo $awsout | jq -r ".${pref[(${c})]}[(${i})].TableName"` | ||
| #echo "cname=$cname" | ||
| #echo $awsout | jq -r ".${pref[(${i})]}[(${i})]" | ||
|
|
||
|
|
||
|
|
||
| rname=${cname//:/_} && rname=${rname//./_} && rname=${rname//\//_} && rname=${rname//&/_} | ||
| echo "$ttft c__${catid}__${dbnam}__${tbnam}__${cname}" | ||
| fn=`printf "%s__c__%s__%s__%s__%s.tf" $ttft $catid ${dbnam} ${tbnam} $rname` | ||
| if [ -f "$fn" ] ; then echo "$fn exists already skipping" && continue; fi | ||
|
|
||
| printf "resource \"%s\" \"c__%s__%s__%s__%s\" {}" $ttft $catid $dbnam ${tbnam} $rname > $fn | ||
|
|
||
|
|
||
| terraform import $ttft.c__${catid}__${dbnam}__${tbnam}__${rname} "${catid}:${dbnam}:${tbnam}:${cname}" | grep Importing | ||
| terraform state show -no-color $ttft.c__${catid}__${dbnam}__${tbnam}__${rname} > t1.txt | ||
|
|
||
| rm -f $fn | ||
|
|
||
|
|
||
| file="t1.txt" | ||
| fl=$(cat $file | wc -l) | ||
| if [[ $fl -eq 0 ]]; then echo "** Empty State show for $dbname ${tbnam} $rname skipping" && continue; fi | ||
|
|
||
| echo $aws2tfmess > $fn | ||
| tarn="" | ||
| inttl=0 | ||
| doneatt=0 | ||
| while IFS= read line | ||
| do | ||
| skip=0 | ||
| # display $line or do something with $line | ||
| t1=`echo "$line"` | ||
| if [[ "$t1" == *"ttl"* ]]; then inttl=1; fi | ||
| if [[ "$t1" == "}" ]]; then inttl=0; fi | ||
|
|
||
| if [[ ${t1} == *"="* ]];then | ||
| tt1=`echo "$line" | cut -f1 -d'=' | tr -d ' '` | ||
| tt2=`echo "$line" | cut -f2- -d'='` | ||
| if [[ ${tt1} == "id" ]];then skip=1; fi | ||
| if [[ ${tt1} == "arn" ]];then skip=1;fi | ||
| if [[ ${tt1} == "owner_id" ]];then skip=1;fi | ||
| if [[ ${tt1} == "creation_time" ]];then skip=1;fi | ||
| if [[ ${tt1} == "last_accessed_time" ]];then skip=1;fi | ||
| if [[ ${tt1} == *"grokPattern"* ]];then skip=1;fi | ||
| fi | ||
|
|
||
| if [ "$skip" == "0" ]; then | ||
| #echo $skip $t1 | ||
| echo "$t1" >> $fn | ||
| fi | ||
|
|
||
| done <"$file" | ||
|
|
||
|
|
||
| done | ||
| fi | ||
| done | ||
|
|
||
| #rm -f t*.txt | ||
|
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,150 @@ | ||
| #!/bin/bash | ||
| if [[ "$1" == "" ]]; then | ||
| echo "must specify catalog id" | ||
| fi | ||
| if [[ "$2" == "" ]]; then | ||
| echo "must specify database name" | ||
| fi | ||
|
|
||
| if [[ "$3" != "" ]]; then | ||
| cmd[0]="$AWS glue get-table --catalog-id $1 --database-name $2 --name $3" | ||
| pref[0]="Table" | ||
| else | ||
| cmd[0]="$AWS glue get-tables --catalog-id $1 --database-name $2" | ||
| pref[0]="TableList" | ||
| fi | ||
|
|
||
| idfilt[0]="Name" | ||
| tft[0]="aws_glue_catalog_table" | ||
|
|
||
| which jq &>/dev/null | ||
| if [[ $? -eq 0 ]]; then | ||
|
|
||
| c=0 | ||
|
|
||
| cm=${cmd[$c]} | ||
| ttft=${tft[(${c})]} | ||
| #echo $cm | ||
| awsout=$(eval $cm 2>/dev/null) | ||
| if [ "$awsout" == "" ]; then | ||
| echo "$cm : You don't have access for this resource" | ||
| echo "PARTITION:NOTABLE99-99" | ||
| exit | ||
| fi | ||
| if [[ "$3" != "" ]]; then | ||
| count=1 | ||
| else | ||
| count=$(echo $awsout | jq ".${pref[(${c})]} | length") | ||
| fi | ||
| if [ "$count" -gt "0" ]; then | ||
| count=$(expr $count - 1) | ||
| for i in $(seq 0 $count); do | ||
| #echo $i | ||
| if [[ "$3" != "" ]]; then | ||
| cname=$(echo $awsout | jq -r ".${pref[(${c})]}.${idfilt[(${c})]}") | ||
| catid=$(echo $awsout | jq -r ".${pref[(${c})]}.CatalogId") | ||
| dbnam=$(echo $awsout | jq -r ".${pref[(${c})]}.DatabaseName") | ||
| else | ||
| cname=$(echo $awsout | jq -r ".${pref[(${c})]}[(${i})].${idfilt[(${c})]}") | ||
| catid=$(echo $awsout | jq -r ".${pref[(${c})]}[(${i})].CatalogId") | ||
| dbnam=$(echo $awsout | jq -r ".${pref[(${c})]}[(${i})].DatabaseName") | ||
| fi | ||
| rname=${cname//:/_} && rname=${rname//./_} && rname=${rname//\//_} && rname=${rname//&/_} | ||
| #echo "$ttft c__${catid}__${dbnam}__${cname}" | ||
| fn=$(printf "%s__c__%s__%s__%s.tf" $ttft $catid ${dbnam} $rname) | ||
| if [ -f "$fn" ]; then echo "$fn exists already skipping" && continue; fi | ||
|
|
||
| printf "resource \"%s\" \"c__%s__%s__%s\" {}" $ttft $catid $dbnam $rname >$fn | ||
|
|
||
| terraform import $ttft.c__${catid}__${dbnam}__${rname} "${catid}:${dbnam}:${cname}" | grep Importing | ||
| terraform state show -no-color $ttft.c__${catid}__${dbnam}__${rname} >t1.txt | ||
|
|
||
| rm -f $fn | ||
|
|
||
| file="t1.txt" | ||
| fl=$(cat $file | wc -l) | ||
| if [[ $fl -eq 0 ]]; then echo "** Empty State show for $dbname $rname skipping" && continue; fi | ||
|
|
||
| echo $aws2tfmess >$fn | ||
| tarn="" | ||
| inttl=0 | ||
| doneatt=0 | ||
| while IFS= read line; do | ||
| skip=0 | ||
| # display $line or do something with $line | ||
| t1=$(echo "$line") | ||
| if [[ "$t1" == *"ttl"* ]]; then inttl=1; fi | ||
| if [[ "$t1" == "}" ]]; then inttl=0; fi | ||
|
|
||
| if [[ ${t1} == *"="* ]]; then | ||
| tt1=$(echo "$line" | cut -f1 -d'=' | tr -d ' ') | ||
| tt2=$(echo "$line" | cut -f2- -d'=') | ||
| if [[ ${tt1} == "id" ]]; then skip=1; fi | ||
| if [[ ${tt1} == "arn" ]]; then skip=1; fi | ||
| if [[ ${tt1} == "owner_id" ]]; then skip=1; fi | ||
| # these are difficult to process so skip for now | ||
| if [[ ${tt1} == *"grokPattern"* ]]; then | ||
|
|
||
| tt2=$(echo $tt2 | sed 's/^"//') | ||
| tt2=$(echo $tt2 | sed 's/"$//') | ||
| tt2=${tt2//\\/\\\\} | ||
| tt2=${tt2//%\{/%%\{} | ||
| tt2=$(echo $tt2 | sed 's/"/\\"/g') | ||
|
|
||
| t1=$(printf "\"grokPattern\" = \"%s\"" "$tt2") | ||
| fi | ||
|
|
||
| if [[ ${tt1} == *"input.format"* ]]; then | ||
|
|
||
| tt2=$(echo $tt2 | sed 's/^"//') | ||
| tt2=$(echo $tt2 | sed 's/"$//') | ||
| tt2=${tt2//\\/\\\\} | ||
| tt2=${tt2//%\{/%%\{} | ||
| tt2=$(echo $tt2 | sed 's/"/\\"/g') | ||
| t1=$(printf "\"input.format\" = \"%s\"" "$tt2") | ||
|
|
||
| fi | ||
| if [[ ${tt1} == *"input.regex"* ]]; then | ||
|
|
||
| tt2=$(echo $tt2 | sed 's/^"//') | ||
| tt2=$(echo $tt2 | sed 's/"$//') | ||
| tt2=${tt2//\\/\\\\} | ||
| tt2=${tt2//%\{/%%\{} | ||
| tt2=$(echo $tt2 | sed 's/"/\\"/g') | ||
| t1=$(printf "\"input.regex\" = \"%s\"" "$tt2") | ||
|
|
||
| fi | ||
| if [[ ${tt1} == "type" ]]; then | ||
| tt2=$(echo "$tt2" | tr -d '"') | ||
| tt2=${tt2//\\/\\\\} | ||
| if [[ ${tt2} == *"struct"* ]]; then | ||
| tt2=$(echo $tt2 | sed 's/^ //') | ||
| #r1=$(echo $RANDOM | md5sum | head -c 20; echo;) | ||
| #gn=`printf "glue-var-%s.tf" $r1` | ||
| #printf "variable \"g-%s\" {\n" $r1 > $gn | ||
| #printf " type = string \n" >> $gn | ||
| #printf " default = \"%s\" \n" $tt2 >> $gn | ||
| #printf "}\n" >> $gn | ||
| #t1=`printf "type = var.g-%s" $r1` | ||
| t1=$(printf "type = \"%s\"" "$tt2") | ||
| fi | ||
| fi | ||
| fi | ||
|
|
||
| if [ "$skip" == "0" ]; then | ||
| #echo $skip $t1 | ||
| echo "$t1" >>$fn | ||
| fi | ||
|
|
||
| done <"$file" | ||
|
|
||
| # get the partitons | ||
| #../../.scripts/get-glue-partition.sh $catid $dbnam $rname | ||
| echo "PARTITION:$rname" | ||
|
|
||
| done # for i | ||
| fi | ||
|
|
||
| fi | ||
| rm -f t*.txt | ||
| cp aws_glue_catalog_table*.tf imported |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,24 @@ | ||
| mydir=$(pwd) | ||
| mydir=$(echo $mydir | rev | cut -f-2 -d'/' | rev) | ||
| which jq &>/dev/null | ||
| if [[ $? -eq 0 ]]; then | ||
| which trivy &>/dev/null | ||
| if [[ $? -eq 0 ]]; then | ||
| ver=$(trivy version | head -1 | cut -f2 -d':' | tr -d ' |.') | ||
| ver=$(expr $ver + 0) | ||
| if [[ $ver -ge 480 ]]; then | ||
| echo "trivy security report" >security-report.txt | ||
| echo "CRITICAL:" >>security-report.txt | ||
| trivy fs --scanners misconfig . -s CRITICAL --format json -q | jq '.Results[].Misconfigurations' | grep -v null | jq '.[] | [.CauseMetadata.Resource, .Description, .References]' 2>/dev/null >>security-report.txt | ||
| echo "HIGH:" >>security-report.txt | ||
| trivy fs --scanners misconfig . -s HIGH --format json -q | jq '.Results[].Misconfigurations' | grep -v null | jq '.[] | [.CauseMetadata.Resource, .Description, .References]' 2>/dev/null >>security-report.txt | ||
| echo "Trivy security report: $mydir/security-report.txt" | ||
| else | ||
| echo "Please upgrade trivy to version v0.48.0 or higher" | ||
| fi | ||
| else | ||
| echo "trivy is not installed. skipping security report" | ||
| fi | ||
| else | ||
| echo "jq is not installed. skipping security report" | ||
| fi |