Cleanup Script – Delete user Staging Data

Delete staging data from user home dir which are older than 10 days

$ cat delete_user_staging_dir.sh

#!/usr/bin/env bash
### delete staging data from user home dir which are older than 10 days ###

IFS=$'\n'

today=`date +'%s'`

echo "---------------------------------------------------------------" >> /home/hdfs/scripts/script9/user_staging_delete_list.txt

hdfs dfs -ls /user/*/.staging | grep "^d" | while read line ; do
dir_date=$(echo ${line} | awk '{print $6}')
difference=$(( ( ${today} - $(date -d ${dir_date} +%s) ) / ( 24*60*60 ) ))
filePath=$(echo ${line} | awk '{print $8}')

if ([ "${difference}" -gt 10 ])
then

hadoop fs -rm -r ${filePath}

laststat=$?
if ([ "${laststat}" == 0 ])
then
echo  "[`date --iso-8601=seconds`] - deleted ${filePath}" >> /home/hdfs/scripts/script9/user_staging_delete_list.txt
fi

fi
done

hadoop fs -rm -r /user/hdfs/.Trash/Current/*