路漫漫其修远兮
吾将上下而求索

k8s学习:日志清理和归档

生产环境,并且在白名单中的项目,30天前的归档,其余清理

要注意find的时候要过滤当前目录  . , 不然会出问题

#!/bin/bash
# 0 1 * * * /bin/bash /data/script/delete_log.sh
#需要将master中的私钥传到每台机器上,公钥传到log机器,能免密登录,白名单要带 -deployment


PATH=/sbin:/bin:/usr/sbin:/usr/bin:/opt/k8s/bin
cd $(dirname $0)

func_delete_history_log(){

   echo
   echo
   echo "$(date +%F_%T) start delete history log"

   back_ip=10.1.1.1
   keep_log_time=30
   k8s_log_dir=/data/k8s-applogs
   graylog_log_dir=/data/graylog
   backup_dir=/data/k8s-log-back
   
   curl -s  http://yum.abc.com/omega/keep_log/white_list > /data/script/white_list 

   for ns in $(kubectl get ns | awk '{print $1}' | grep prod); do 
       if [[ -d $k8s_log_dir/$ns ]]; then
           for deployment in $(ls $k8s_log_dir/$ns); do
               cat /data/script/white_list | grep -q $deployment
               if [[ $? -eq 0 ]]; then
                   if [[ -d $k8s_log_dir/$ns/$deployment ]]; then
                       cd $k8s_log_dir/$ns/$deployment                 
                       for pod in $(find  -maxdepth 1 -mtime +$keep_log_time | grep -v ^.$); do 
                           if [[ -d $pod ]]; then  
                               echo "start tar $k8s_log_dir/$ns/$deployment/$pod"
                               cd $pod
                               for log in $(ls); do
                                   tar zcf ${log}.tar.gz $log
                                   /bin/rm -rf $log
                               done

                               cd $k8s_log_dir/$ns/$deployment
                               ssh $back_ip "mkdir -p $backup_dir/$ns/$deployment"
                               scp -o  StrictHostKeyChecking=no -r $pod $back_ip:$backup_dir/$ns/$deployment
                               echo "start rm $k8s_log_dir/$ns/$deployment/$pod"
                               /bin/rm -rf $pod
                           fi
                       done
                   fi
               else 
                   if [[ -d $k8s_log_dir/$ns/$deployment ]]; then
                       cd $k8s_log_dir/$ns/$deployment                 
                       for pod in $(find  -maxdepth 1 -mtime +$keep_log_time | grep -v ^.$); do 
                           rm -rf $pod
                           echo "start rm $k8s_log_dir/$ns/$deployment/$pod"
                       done
                   fi
               fi
           done
       fi
   done     


   for ns in $(kubectl get ns | awk '{print $1}' | egrep "qua$|daily$|tmp$|pre$"); do 
       if [[ -d $k8s_log_dir/$ns ]]; then
           for deployment in $(ls $k8s_log_dir/$ns); do
               
               if [[ -d $k8s_log_dir/$ns/$deployment ]]; then
                   cd $k8s_log_dir/$ns/$deployment                 
                   for pod in $(find  -maxdepth 1 -mtime +$keep_log_time | grep -v ^.$); do 
                       rm -rf $pod
                       echo "start rm $k8s_log_dir/$ns/$deployment/$pod"
                   done
               fi
           done
       fi
   done     


   for ns in $(kubectl get ns | awk '{print $1}' | grep "prod"); do 
       if [[ -d $graylog_log_dir/$ns ]]; then
           for deployment in $(ls $graylog_log_dir/$ns); do
               
               if [[ -d $graylog_log_dir/$ns/$deployment ]]; then
                   cd $graylog_log_dir/$ns/$deployment                 
                   for pod in $(find  -maxdepth 1 -mtime +3 | grep -v ^.$); do 
                       rm -rf $pod
                       echo "start rm $graylog_log_dir/$ns/$deployment/$pod"
                   done
               fi
           done
       fi
   done     


   echo "$(date +%F_%T) stop delete log"


}

func_delete_history_log &>> delete_log.log

生产都备份,白名单永久,非白名单40天

#!/bin/bash
# 0 1 * * * /bin/bash /data/script/delete_log.sh
#需要将master中的私钥传到每台机器上,公钥传到log机器,能免密登录,白名单要带 -deployment


PATH=/sbin:/bin:/usr/sbin:/usr/bin:/opt/k8s/bin
cd $(dirname $0)

func_delete_history_log(){

    echo
    echo
    echo "$(date +%F_%T) start delete history log"

    back_ip=10.216.35.103
    keep_log_time=30
    k8s_log_dir=/data/k8s-applogs
    graylog_log_dir=/data/graylog
    backup_dir=/logback/prod_whitelist_log
    prod_dir=/logback/prod_app_log
    
    curl -s  http://yum.andy.com/omega/keep_log/white_list > /data/script/white_list 
    kubectl get pod --all-namespaces | egrep "qua|daily|tmp|pre|prod" | awk '{print $2}'> delete_log.tmp

    for ns in $(kubectl get ns | awk '{print $1}' | grep prod); do 
        if [[ -d $k8s_log_dir/$ns ]]; then
            for deployment in $(ls $k8s_log_dir/$ns); do
                cat /data/script/white_list | grep -q $deployment
                if [[ $? -eq 0 ]]; then
                    if [[ -d $k8s_log_dir/$ns/$deployment ]]; then
                        cd $k8s_log_dir/$ns/$deployment
                        for pod in $(find  -maxdepth 1 -mtime +$keep_log_time | grep -v ^.$); do 
                            podname=$(echo $pod | awk -F '/' '{print $2}')
                            grep -q $podname /data/script/delete_log.tmp
                            if [[ -d $pod && $? -eq 1 ]]; then  
                                echo "start tar whitelist $k8s_log_dir/$ns/$deployment/$pod"
                                cd $pod
                                for log in $(ls); do
                                    tar zcf ${log}.tar.gz $log
                                    /bin/rm -rf $log
                                done

                                cd $k8s_log_dir/$ns/$deployment
                                ssh $back_ip "mkdir -p $backup_dir/$ns/$deployment"
                                scp -o StrictHostKeyChecking=no -r $pod $back_ip:$backup_dir/$ns/$deployment
                                echo "start rm whitelist $k8s_log_dir/$ns/$deployment/$pod"
                                /bin/rm -rf $pod
                            fi
                        done
                    fi
                else 
                    if [[ -d $k8s_log_dir/$ns/$deployment ]]; then
                        cd $k8s_log_dir/$ns/$deployment
                        for pod in $(find  -maxdepth 1 -mtime +$keep_log_time | grep -v ^.$); do 
                            podname=$(echo $pod | awk -F '/' '{print $2}')
                            grep -q $podname /data/script/delete_log.tmp
                            if [[ -d $pod && $? -eq 1 ]]; then  
                                echo "start tar prod $k8s_log_dir/$ns/$deployment/$pod"
                                cd $pod
                                for log in $(ls); do
                                    tar zcf ${log}.tar.gz $log
                                    /bin/rm -rf $log
                                done

                                cd $k8s_log_dir/$ns/$deployment
                                ssh $back_ip "mkdir -p $prod_dir/$ns/$deployment"
                                scp -o StrictHostKeyChecking=no -r $pod $back_ip:$prod_dir/$ns/$deployment
                                echo "start rm prod $k8s_log_dir/$ns/$deployment/$pod"
                                /bin/rm -rf $pod
                            fi
                        done
                    fi
                fi
            done
        fi
    done     


    for ns in $(kubectl get ns | awk '{print $1}' | egrep "qua$|daily$|tmp$|pre$"); do 
        if [[ -d $k8s_log_dir/$ns ]]; then
            for deployment in $(ls $k8s_log_dir/$ns); do
                
                if [[ -d $k8s_log_dir/$ns/$deployment ]]; then
                    cd $k8s_log_dir/$ns/$deployment                 
                    for pod in $(find  -maxdepth 1 -mtime +$keep_log_time | grep -v ^.$); do 
                        podname=$(echo $pod | awk -F '/' '{print $2}')
                        grep -q $podname /data/script/delete_log.tmp
                        if [[ -d $pod && $? -eq 1 ]]; then  
                            rm -rf $pod
                            echo "start rm $k8s_log_dir/$ns/$deployment/$pod"
                        fi
                    done
                fi
            done
        fi
    done     


    for ns in $(kubectl get ns | awk '{print $1}' | grep "prod"); do 
        if [[ -d $graylog_log_dir/$ns ]]; then
            for deployment in $(ls $graylog_log_dir/$ns); do
                
                if [[ -d $graylog_log_dir/$ns/$deployment ]]; then
                    cd $graylog_log_dir/$ns/$deployment                 
                    for pod in $(find  -maxdepth 1 -mtime +3 | grep -v ^.$); do 
                        podname=$(echo $pod | awk -F '/' '{print $2}')
                        grep -q $podname /data/script/delete_log.tmp
                        if [[ -d $pod && $? -eq 1 ]]; then  
                            rm -rf $pod
                            echo "start rm $graylog_log_dir/$ns/$deployment/$pod"
                        fi
                    done
                fi
            done
        fi
    done     


    echo "$(date +%F_%T) stop delete log"


}

func_delete_history_log &>> delete_log.log

宿主机只保留5天时间的日志,其他的都传到日志服务器,从web上查看日志

#!/bin/bash
# 0 1 * * * /bin/bash /data/script/delete_log.sh
#需要将master中的私钥传到每台机器上,公钥传到log机器,能免密登录


PATH=/sbin:/bin:/usr/sbin:/usr/bin:/opt/k8s/bin
cd $(dirname $0)

func_delete_history_log(){

    echo
    echo
    echo "$(date +%F_%T) start delete history log"

    back_ip=10.216.35.103
    keep_log_time=5
    k8s_log_dir=/data/k8s-applogs
    graylog_log_dir=/data/graylog
    backup_dir=/logback/prod_app_log
    datetime=$(date +%Y%m%d --date='-7 day')

    for ns in $(kubectl get ns | awk '{print $1}' | grep prod); do 
        if [[ -d $k8s_log_dir/$ns ]]; then
            for deployment in $(ls $k8s_log_dir/$ns); do
                if [[ -d $k8s_log_dir/$ns/$deployment ]]; then
                    cd $k8s_log_dir/$ns/$deployment
                    for pod in $(find -maxdepth 1 | grep -v ^.$); do
                        podname=$(echo $pod | awk -F '/' '{print $2}')
                        if [[ -d $pod ]]; then
                            cd $pod                       
                            for log in $(find -maxdepth 1 -mtime +$keep_log_time | grep -v ^.$); do
                                logname=$(echo $log | awk -F '/' '{print $2}')
                                
                                echo "start tar log file $k8s_log_dir/$ns/$deployment/$pod/$logname"
                                tar zcf ${podname}-${logname}.tar.gz $logname
                                
                                ssh $back_ip "mkdir -p $backup_dir/$ns/$deployment/$datetime/"
                                scp -o StrictHostKeyChecking=no -r ${podname}-${logname}.tar.gz $back_ip:$backup_dir/$ns/$deployment/$datetime/
                                
                                
                                [[ $? -eq 0 ]] && /bin/rm -rf $logname ${podname}-${logname}.tar.gz
                                echo "start rm log $k8s_log_dir/$ns/$deployment/$pod/$logname"
                            done
                            cd ..
                        fi
                    done
                fi
            done
        fi
    done     


    for ns in $(kubectl get ns | awk '{print $1}' | egrep "qua$|daily$|tmp$|pre$"); do 
         if [[ -d $k8s_log_dir/$ns ]]; then
            for deployment in $(ls $k8s_log_dir/$ns); do
                if [[ -d $k8s_log_dir/$ns/$deployment ]]; then
                    cd $k8s_log_dir/$ns/$deployment
                    for pod in $(find -maxdepth 1 | grep -v ^.$); do
                        podname=$(echo $pod | awk -F '/' '{print $2}')
                        if [[ -d $pod ]]; then
                            cd $pod                         
                            for log in $(find -maxdepth 1 -mtime +$keep_log_time | grep -v ^.$); do
                                logname=$(echo $log | awk -F '/' '{print $2}')
                                
                                echo "start rm log $k8s_log_dir/$ns/$deployment/$pod/$logname"
                                /bin/rm -rf $logname
                            done
                            cd ..
                        fi
                    done
                fi
            done
        fi
    done     


    for ns in $(kubectl get ns | awk '{print $1}'); do 
         if [[ -d $graylog_log_dir/$ns ]]; then
            for deployment in $(ls $graylog_log_dir/$ns); do
                if [[ -d $graylog_log_dir/$ns/$deployment ]]; then
                    cd $graylog_log_dir/$ns/$deployment
                    for pod in $(find -maxdepth 1 | grep -v ^.$); do
                        podname=$(echo $pod | awk -F '/' '{print $2}')
                        if [[ -d $pod ]]; then
                            cd $pod                         
                            for log in $(find -maxdepth 1 -mtime +2 | grep -v ^.$); do
                                logname=$(echo $log | awk -F '/' '{print $2}')
                                
                                echo "start rm graylog $graylog_log_dir/$ns/$deployment/$pod/$logname"
                                /bin/rm -rf $logname
                            done
                            cd ..
                        fi
                    done
                fi
            done
        fi
    done     


    echo "$(date +%F_%T) stop delete log"


}

func_delete_history_log &>> delete_log.log

未经允许不得转载:江哥架构师笔记 » k8s学习:日志清理和归档

分享到:更多 ()

评论 抢沙发

评论前必须登录!