Cleanup Script – Delete flink application entries from hdfs

Delete flink appid dirs from hdfs which are older than 10 days
$ cat delete_hdfs_flink_appid.sh


#!/usr/bin/env bash
### delete flink appid dirs from hdfs which are older than 10 days ###

IFS=$'\n'

today=`date +'%s'`

echo "---------------------------------------------------------------" >> /home/hdfs/scripts/script9/flink_delete_list.txt

hdfs dfs -ls /user/hdfs/.flink | grep "^d" | while read line ; do
dir_date=$(echo ${line} | awk '{print $6}')
difference=$(( ( ${today} - $(date -d ${dir_date} +%s) ) / ( 24*60*60 ) ))
filePath=$(echo ${line} | awk '{print $8}')

if ([ "${difference}" -gt 10 ])
then

hadoop fs -rm -r ${filePath}

laststat=$?
if ([ "${laststat}" == 0 ])
then
echo  "[`date --iso-8601=seconds`] - deleted ${filePath}" >> /home/hdfs/scripts/script9/flink_delete_list.txt
fi

fi
done

hadoop fs -rm -r /user/hdfs/.Trash/Current/*
0 Comments

There are no comments yet

Leave a comment

Your email address will not be published. Required fields are marked *