diff --git a/.scripts/get-glue-partition.sh b/.scripts/get-glue-partition.sh new file mode 100755 index 0000000..c810ce3 --- /dev/null +++ b/.scripts/get-glue-partition.sh @@ -0,0 +1,123 @@ +#!/bin/bash +if [[ "$1" == "" ]]; then + echo "must specify catalog id" + exit +fi +if [[ "$2" == "" ]]; then + echo "must specify database name" + exit +fi +if [[ "$3" == "" ]]; then + echo "must specify tablee name" + exit +fi + +cmd[0]="$AWS glue get-partitions --catalog-id $1 --database-name $2 --table-name $3" +pref[0]="Partitions" + +#if [[ "$4" != "" ]]; then +# cmd[0]="$AWS glue get-partition --catalog-id $1 --database-name $2 --table-name $3 --partition-values $4" +# pref[0]="Partition" +#else +# cmd[0]="$AWS glue get-partitions --catalog-id $1 --database-name $3 --table-name $3" +# pref[0]="Partitions" +#fi + +idfilt[0]="Values" +tft[0]="aws_glue_partition" + +#pks=$(cat $tfa.json | jq .values.partition_keys) +#pcount=`echo $pks | jq ". | length"` +#if [ "$pcount" -gt "0" ]; then +# pcount=`expr $pcount - 1` +# for i in `seq 0 $pcount`; do +# tp=`echo $pks | jq -r ".[(${i})].name"` +# echo "partition=$tp" +# ../../scripts/get-glue-partition.sh $catid $dbnam $rname $tp +# done + #fi + + +for c in `seq 0 0`; do + + cm=${cmd[$c]} + ttft=${tft[(${c})]} + #echo $cm + awsout=`eval $cm 2> /dev/null` + if [ "$awsout" == "" ];then + echo "$cm : You don't have access for this resource" + exit + fi + + count=`echo $awsout | jq ".${pref[(${c})]} | length"` + #echo "found $count partitions" + + if [ "$count" -gt "0" ]; then + count=`expr $count - 1` + for i in `seq 0 $count`; do + #echo $i + + cname=`echo $awsout | jq -r ".${pref[(${c})]}[(${i})].${idfilt[(${c})]}[(${c})]"` + catid=`echo $awsout | jq -r ".${pref[(${c})]}[(${i})].CatalogId"` + dbnam=`echo $awsout | jq -r ".${pref[(${c})]}[(${i})].DatabaseName"` + tbnam=`echo $awsout | jq -r ".${pref[(${c})]}[(${i})].TableName"` + #echo "cname=$cname" + #echo $awsout | jq -r ".${pref[(${i})]}[(${i})]" + + + + rname=${cname//:/_} && rname=${rname//./_} && rname=${rname//\//_} && rname=${rname//&/_} + echo "$ttft c__${catid}__${dbnam}__${tbnam}__${cname}" + fn=`printf "%s__c__%s__%s__%s__%s.tf" $ttft $catid ${dbnam} ${tbnam} $rname` + if [ -f "$fn" ] ; then echo "$fn exists already skipping" && continue; fi + + printf "resource \"%s\" \"c__%s__%s__%s__%s\" {}" $ttft $catid $dbnam ${tbnam} $rname > $fn + + + terraform import $ttft.c__${catid}__${dbnam}__${tbnam}__${rname} "${catid}:${dbnam}:${tbnam}:${cname}" | grep Importing + terraform state show -no-color $ttft.c__${catid}__${dbnam}__${tbnam}__${rname} > t1.txt + + rm -f $fn + + + file="t1.txt" + fl=$(cat $file | wc -l) + if [[ $fl -eq 0 ]]; then echo "** Empty State show for $dbname ${tbnam} $rname skipping" && continue; fi + + echo $aws2tfmess > $fn + tarn="" + inttl=0 + doneatt=0 + while IFS= read line + do + skip=0 + # display $line or do something with $line + t1=`echo "$line"` + if [[ "$t1" == *"ttl"* ]]; then inttl=1; fi + if [[ "$t1" == "}" ]]; then inttl=0; fi + + if [[ ${t1} == *"="* ]];then + tt1=`echo "$line" | cut -f1 -d'=' | tr -d ' '` + tt2=`echo "$line" | cut -f2- -d'='` + if [[ ${tt1} == "id" ]];then skip=1; fi + if [[ ${tt1} == "arn" ]];then skip=1;fi + if [[ ${tt1} == "owner_id" ]];then skip=1;fi + if [[ ${tt1} == "creation_time" ]];then skip=1;fi + if [[ ${tt1} == "last_accessed_time" ]];then skip=1;fi + if [[ ${tt1} == *"grokPattern"* ]];then skip=1;fi + fi + + if [ "$skip" == "0" ]; then + #echo $skip $t1 + echo "$t1" >> $fn + fi + + done <"$file" + + + done + fi +done + +#rm -f t*.txt + diff --git a/.scripts/get-glue-table.sh b/.scripts/get-glue-table.sh new file mode 100755 index 0000000..9ef9350 --- /dev/null +++ b/.scripts/get-glue-table.sh @@ -0,0 +1,150 @@ +#!/bin/bash +if [[ "$1" == "" ]]; then + echo "must specify catalog id" +fi +if [[ "$2" == "" ]]; then + echo "must specify database name" +fi + +if [[ "$3" != "" ]]; then + cmd[0]="$AWS glue get-table --catalog-id $1 --database-name $2 --name $3" + pref[0]="Table" +else + cmd[0]="$AWS glue get-tables --catalog-id $1 --database-name $2" + pref[0]="TableList" +fi + +idfilt[0]="Name" +tft[0]="aws_glue_catalog_table" + +which jq &>/dev/null +if [[ $? -eq 0 ]]; then + + c=0 + + cm=${cmd[$c]} + ttft=${tft[(${c})]} + #echo $cm + awsout=$(eval $cm 2>/dev/null) + if [ "$awsout" == "" ]; then + echo "$cm : You don't have access for this resource" + echo "PARTITION:NOTABLE99-99" + exit + fi + if [[ "$3" != "" ]]; then + count=1 + else + count=$(echo $awsout | jq ".${pref[(${c})]} | length") + fi + if [ "$count" -gt "0" ]; then + count=$(expr $count - 1) + for i in $(seq 0 $count); do + #echo $i + if [[ "$3" != "" ]]; then + cname=$(echo $awsout | jq -r ".${pref[(${c})]}.${idfilt[(${c})]}") + catid=$(echo $awsout | jq -r ".${pref[(${c})]}.CatalogId") + dbnam=$(echo $awsout | jq -r ".${pref[(${c})]}.DatabaseName") + else + cname=$(echo $awsout | jq -r ".${pref[(${c})]}[(${i})].${idfilt[(${c})]}") + catid=$(echo $awsout | jq -r ".${pref[(${c})]}[(${i})].CatalogId") + dbnam=$(echo $awsout | jq -r ".${pref[(${c})]}[(${i})].DatabaseName") + fi + rname=${cname//:/_} && rname=${rname//./_} && rname=${rname//\//_} && rname=${rname//&/_} + #echo "$ttft c__${catid}__${dbnam}__${cname}" + fn=$(printf "%s__c__%s__%s__%s.tf" $ttft $catid ${dbnam} $rname) + if [ -f "$fn" ]; then echo "$fn exists already skipping" && continue; fi + + printf "resource \"%s\" \"c__%s__%s__%s\" {}" $ttft $catid $dbnam $rname >$fn + + terraform import $ttft.c__${catid}__${dbnam}__${rname} "${catid}:${dbnam}:${cname}" | grep Importing + terraform state show -no-color $ttft.c__${catid}__${dbnam}__${rname} >t1.txt + + rm -f $fn + + file="t1.txt" + fl=$(cat $file | wc -l) + if [[ $fl -eq 0 ]]; then echo "** Empty State show for $dbname $rname skipping" && continue; fi + + echo $aws2tfmess >$fn + tarn="" + inttl=0 + doneatt=0 + while IFS= read line; do + skip=0 + # display $line or do something with $line + t1=$(echo "$line") + if [[ "$t1" == *"ttl"* ]]; then inttl=1; fi + if [[ "$t1" == "}" ]]; then inttl=0; fi + + if [[ ${t1} == *"="* ]]; then + tt1=$(echo "$line" | cut -f1 -d'=' | tr -d ' ') + tt2=$(echo "$line" | cut -f2- -d'=') + if [[ ${tt1} == "id" ]]; then skip=1; fi + if [[ ${tt1} == "arn" ]]; then skip=1; fi + if [[ ${tt1} == "owner_id" ]]; then skip=1; fi + # these are difficult to process so skip for now + if [[ ${tt1} == *"grokPattern"* ]]; then + + tt2=$(echo $tt2 | sed 's/^"//') + tt2=$(echo $tt2 | sed 's/"$//') + tt2=${tt2//\\/\\\\} + tt2=${tt2//%\{/%%\{} + tt2=$(echo $tt2 | sed 's/"/\\"/g') + + t1=$(printf "\"grokPattern\" = \"%s\"" "$tt2") + fi + + if [[ ${tt1} == *"input.format"* ]]; then + + tt2=$(echo $tt2 | sed 's/^"//') + tt2=$(echo $tt2 | sed 's/"$//') + tt2=${tt2//\\/\\\\} + tt2=${tt2//%\{/%%\{} + tt2=$(echo $tt2 | sed 's/"/\\"/g') + t1=$(printf "\"input.format\" = \"%s\"" "$tt2") + + fi + if [[ ${tt1} == *"input.regex"* ]]; then + + tt2=$(echo $tt2 | sed 's/^"//') + tt2=$(echo $tt2 | sed 's/"$//') + tt2=${tt2//\\/\\\\} + tt2=${tt2//%\{/%%\{} + tt2=$(echo $tt2 | sed 's/"/\\"/g') + t1=$(printf "\"input.regex\" = \"%s\"" "$tt2") + + fi + if [[ ${tt1} == "type" ]]; then + tt2=$(echo "$tt2" | tr -d '"') + tt2=${tt2//\\/\\\\} + if [[ ${tt2} == *"struct"* ]]; then + tt2=$(echo $tt2 | sed 's/^ //') + #r1=$(echo $RANDOM | md5sum | head -c 20; echo;) + #gn=`printf "glue-var-%s.tf" $r1` + #printf "variable \"g-%s\" {\n" $r1 > $gn + #printf " type = string \n" >> $gn + #printf " default = \"%s\" \n" $tt2 >> $gn + #printf "}\n" >> $gn + #t1=`printf "type = var.g-%s" $r1` + t1=$(printf "type = \"%s\"" "$tt2") + fi + fi + fi + + if [ "$skip" == "0" ]; then + #echo $skip $t1 + echo "$t1" >>$fn + fi + + done <"$file" + + # get the partitons + #../../.scripts/get-glue-partition.sh $catid $dbnam $rname + echo "PARTITION:$rname" + + done # for i + fi + +fi +rm -f t*.txt +cp aws_glue_catalog_table*.tf imported diff --git a/.scripts/trivy-check.sh b/.scripts/trivy-check.sh new file mode 100755 index 0000000..b9d7d98 --- /dev/null +++ b/.scripts/trivy-check.sh @@ -0,0 +1,24 @@ +mydir=$(pwd) +mydir=$(echo $mydir | rev | cut -f-2 -d'/' | rev) +which jq &>/dev/null +if [[ $? -eq 0 ]]; then + which trivy &>/dev/null + if [[ $? -eq 0 ]]; then + ver=$(trivy version | head -1 | cut -f2 -d':' | tr -d ' |.') + ver=$(expr $ver + 0) + if [[ $ver -ge 480 ]]; then + echo "trivy security report" >security-report.txt + echo "CRITICAL:" >>security-report.txt + trivy fs --scanners misconfig . -s CRITICAL --format json -q | jq '.Results[].Misconfigurations' | grep -v null | jq '.[] | [.CauseMetadata.Resource, .Description, .References]' 2>/dev/null >>security-report.txt + echo "HIGH:" >>security-report.txt + trivy fs --scanners misconfig . -s HIGH --format json -q | jq '.Results[].Misconfigurations' | grep -v null | jq '.[] | [.CauseMetadata.Resource, .Description, .References]' 2>/dev/null >>security-report.txt + echo "Trivy security report: $mydir/security-report.txt" + else + echo "Please upgrade trivy to version v0.48.0 or higher" + fi + else + echo "trivy is not installed. skipping security report" + fi +else + echo "jq is not installed. skipping security report" +fi