Delete staging data from user home dir which are older than 10 days
$ cat delete_user_staging_dir.sh
#!/usr/bin/env bash
### delete staging data from user home dir which are older than 10 days ###
IFS=$'\n'
today=`date +'%s'`
echo "---------------------------------------------------------------" >> /home/hdfs/scripts/script9/user_staging_delete_list.txt
hdfs dfs -ls /user/*/.staging | grep "^d" | while read line ; do
dir_date=$(echo ${line} | awk '{print $6}')
difference=$(( ( ${today} - $(date -d ${dir_date} +%s) ) / ( 24*60*60 ) ))
filePath=$(echo ${line} | awk '{print $8}')
if ([ "${difference}" -gt 10 ])
then
hadoop fs -rm -r ${filePath}
laststat=$?
if ([ "${laststat}" == 0 ])
then
echo "[`date --iso-8601=seconds`] - deleted ${filePath}" >> /home/hdfs/scripts/script9/user_staging_delete_list.txt
fi
fi
done
hadoop fs -rm -r /user/hdfs/.Trash/Current/*
0 Comments