# BLOG://renwole.com/ #The following scripts need to be executed in batch by using a springboard machine or a password free login machine, but can also be modified to be executed on a single machine. Just note out the login phase # tite=$(date "+%Y-%m-%d %H:%M:%S") #Hostnamelist="Host file list, one IP per line, without double quotation marks, for example;/mnt/renwolecom" hostnamelist="/tmp/renwole_com" #Clearing failed logs renwole_com_disk_clean_log="/tmp/renwole_clean_failure.log" #for envNamelist in ${Name[@]};  do #If you use the api interface to get the host list, you can use jq to filter josn, but first install jq 1.5 and above, yum install epel&&yum install jq, the default version of Centos 7 for epel source is 1.5+, and the Centos6+version is 1.3 #Curl - s can obtain the josn format through url and filter through jq | jq. data | jq - r [].hostname #done >$hostnamelist #The nailing notification can specify multiple unqualified notifications according to the if condition dingding_Notice() { dingtalk_openapi_url="//oapi.dingtalk.com" dingtalk_openapi_token="Token" if [[ "$(disk_use)" -ge 80 ]];  then curl ''$dingtalk_openapi_url'/robot/send?access_token='$dingtalk_openapi_token'' \ -H 'Content-Type: application/json' \ -d '{"msgtype": "text", "text": { "Content": "The system log has been cleared successfully, but the used space still exceeds 80%. Please repair it in time. The machine is: '" $hostname "'" } }' fi } #Check whether the machine can log in. Try 5 times at most and wait for 5 seconds each time (if the machine is extremely stuck, failure to retry may cause false alarms) check_login() { for i in $(seq 1 5);  do [ $i -gt 1 ] $login_docker 2> /dev/null exit && antl=0 && break || antl=$?; sleep 5 done echo $antl } #After logging in to the container, check whether the relevant commands are available, and retry 3 times in case of failure (some machines are extremely stuck, and will timeout or retry when getting the results) check_login_command() { for i in $(seq 1 3);  do [[ $i -gt 1 ]] $login_docker >/dev/null 2>&1 "type du" >/dev/null 2>&1 && antc=0 && break || antc=$?; sleep 5 done echo $antc } #Check whether the machine can be connected to the network normally. Retry 5 times if failure occurs, and wait 10 seconds for each retry (optional) check_login_command_netwotk() { #If the network is abnormal, please check this file on the container machine gateway="www.renwole.com" curl_timeout="curl -I -s --connect-timeout 16" for i in $(seq 1 5);  do [[ $i -gt 1 ]] $login_docker "$curl_timeout $gateway 2>/dev/null -w %{http_code} | tail -n1" && antn=200 && break || antn="$?"; sleep 10 done } #Get the current disk usage size remotely disk_use() { disk_uses=$($login_docker "df -P|sort -n|grep /dev |grep -v -E '(shm|tmp|boot)'") disk_status=$(echo $disk_uses | awk -F'[ %]+' '{print $5}') echo $disk_status } #Find the log directory and file type to be cleaned, and specify the size to be cleaned. Here, specify 20M system_log_file() { #Multiple directory patrols are supported, separated by spaces, and the '/' must be followed by renwole_log_dir=(/var/log/ /usr/local/nginx/logs/) for log_dirs in ${renwole_log_dir[*]};  do #Log in to the machine remotely, define the file type to be cleaned, and then throw it into the black hole $login_docker "find $log_dirs 2> /dev/null -type f -size +20M \( -name "*.data*" \ -o -name "message*" -o -iname "wtm*" -o -name "vsa*" -o -name "secu*" \ -o -name "cron*" -o -name "*.log*" \) -exec cp /dev/null {} \; " done } #Get which directories or files occupy large space, sort by size, and get the first two lines of file size directories system_file_size_check() { home_file_size=$($login_docker "du -h --max-depth=5 /home/* | sort -h | tail -n5 |head -n2") echo $home_file_size } #Pipeline token | Process concurrency renwole_com="/tmp/renwolcomfile" [ -e "$renwole_com" ] || mkfifo $renwole_com exec 3<>$renwole_com rm -rf $renwole_com for ((i=1; i<=1000;i++)); do echo >&3 done #Start to cycle check the use status of machine disks and disks. If the conditions are not met, an alarm will be given for hostname in `cat $hostnamelist`; do read -u3 { #Remotely detect whether the machine can be logged in. The maximum attempt is 30s. Machines that cannot log in without password will automatically skip the login login_docker="timeout 30 ssh -o BatchMode=yes $hostname" if [[ $(check_login) -ne 0 ]];  then Echo "Container login failed $hostname">>$renwole_com_disk_clean_log #If conditions are met, terminate this cycle and continue the next cycle echo >&3 continue  fi if [[ $(check_login_command) -ne 0 ]];  then Echo "Cannot find du command $hostname">>$renwole_com_disk_clean_log echo >&3 continue fi if [[ $(disk_use)  -gt 80 ]];  then #If the disk utilization rate is greater than 80%, trigger cleaning system_log_file else Echo "The current disk usage $(disk_use)% does not exceed 80%, $hostname" echo >&3 continue fi if [[ $(disk_use)  -gt 80 ]];  then Echo "$title $hostname Disk cleaning failed. The current usage rate is $(disk_use)%. File size: $(system_file_size_check)">>$renwole_com_disk_clean_log Echo "Failed to clean the disk. The current usage rate is $(disk_use)% $hostname" else Echo "The disk is cleaned successfully. The usage rate after cleaning is $(disk_use)% $hostname" fi echo >&3 }& done wait #Close exec 3<&- exec 3>&- #Whether to enable nailing notification, which is closed by default # dingding_Notice