while(motivation <= 0)

I spent the better part of month working on and off on this little script. I never could get the part of that looked for missing files it to work right. It turns out that there can be considerable difference between

if [] if [[]] if [ $() ];

At the end of the day one of my guys figured it out.

#!/bin/bash
secretobj=$(aws secretsmanager get-secret-value --secret-id "product/service/reporting" --region us-east-1)
rptpassword=$(echo $secretobj | jq ".SecretString" | sed 's/\\//g' | sed -E 's/^"//; s/"$//' | jq ".password" | sed -E 's/^"//; s/"$//' )
logfilename="service-report-log_$(date +'%Y-%m-%d').log"
echo "starting report run $(date '+%Y%m%d')" > "$logfilename"
ls service-reports | sort  > existingfiles.txt
allfiles=$(sshpass -p "$rptpassword" sftp -oBatchMode=no -b - service-reports << !
cd ppreports/outgoing
ls
exit
!
)
#cleanup output from sftp server
echo "$allfiles" | sed -e '1,/^sftp> ls$/d' -e '/^sftp> cd ppreports\/outgoing$/d' -e '/^sftp> exit/d' | sort > allfiles.txt

echo "all files on server:\n $allfiles" >> "$logfilename"

#nuke the files to grab list if it exists.
if [ -f filestograb.txt ]; then
        rm filestograb.txt
fi

while IFS= read -r filename
do
        if ! grep -qxF $filename existingfiles.txt; then
                echo $filename >> filestograb.txt
        fi
done < allfiles.txt

comm -23 allfiles.txt existingfiles.txt > filestograb2.txt
if [ -f filestograb.txt ]; then

        echo "files to grab:\n" >> "$logfilename"
        cat filestograb.txt >> "$logfilename"
        echo "starting scp copy\n"
        while IFS= read -r filename
        do
                sshpass -p "$rptpassword" scp service-reports:ppreports/outgoing/$filename service-reports/$filename

        done < filestograb.txt
else
        echo "No files missing?"
fi
echo "ending report run $(date '+%Y%m%d')" >> "$logfilename"