while(motivation <= 0)

So this week has been a lot. I can’t talk about most of it but there was a lot of good bash script to come out of it. Today also marks the end of the road for one of my favorite people on this planet, my Grandpa CL. May he rest in piece. This is for test best as he wasn’t doing well and If I somehow make it to 99, I hope I can go out in my own home just like him. So back to this weeks work:
New to me this week:

• Reading load balancer logs
• Sed on Mac is different than GNU Sed. Props to homebrew for gsed to mac.
• Finding and cat’ing all files recursively in folders
• Using awk to count things
• doing math on the command line
• merge files line by line with paste
• wc -l #count the lines in a file
• du -sh path to dir to size

#grab the file list from all the s3 buckets:
#!/bin/bash
echo "Searching"
while IFS= read -r bucket
do
	if  [ ! -f "bucket_contents/$bucket.txt" ]; then

		echo "Grabbing bucket: $bucket"
		aws s3 ls "s3://$bucket" --recursive --profile events > "bucket_contents/$bucket.txt"
	fi
	
done < gs_buckets.txt

#Loop through buckets and grep for matching file names.
#!/bin/bash
echo "Searching"
while IFS= read -r bucket
do
if [ -f "bucket_contents/$bucket.txt" ]; then

echo "Searching bucket: $bucket"
while IFS= read -r filename
do
cat "bucket_contents/$bucket.txt" | gsed -E 's/^[0-9-]+\s+[0-9:]+\s+[0-9]+\s+//' | grep -e "$filename" >>filelog.txt
done < filelist.txt
fi
done < gs_buckets.txt

#Mac Sed != Linux Sed

#Cat all of the files in all of the folders
find . -type f -exec cat {} + 

#Read all files from an aws load balancer, return code 200
 find . -type f -exec cat {} +  | awk '{print $10 ","  $12 "," $13}' | grep -w 200 | sort | uniq -c | sort | awk '{print $1 "," $2}' > ../test.csv

#Take the port number off an ip address
gsed -E 's/:[0-9]+,/,/'

# Use AWK to count the number of lines between two numbers in a file.
#!/bin/bash
if [ $# -eq 3 ]; then
	topNo=$1
	botNo=$2
	total=$3
	date
	echo "Remaining:"
	left=$(awk -F'/' '{print $2}' movedir.txt | \ 
	awk -v top="$topNo" -v bottom="$botNo" '$1 > bottom && $1 <= top {count++} END {print count}')
	echo "There are $left directories left to process"
	#Do math and output the result
	pctleft=$(bc -l <<< "1-($left/$total)")
	echo "$pctleft complete"

else
	echo "(top no) (bottom no) (total)"
fi