PHP Classes

File: ip2asn_purger.sh

Recommend this page to a friend!
  Classes of Peter Kahl  >  PHP IP to ASN Mapping  >  ip2asn_purger.sh  >  Download  
File: ip2asn_purger.sh
Role: Auxiliary data
Content type: text/plain
Description: Auxiliary data
Class: PHP IP to ASN Mapping
Map IPv4 or IPv6 address to ASN
Author: By
Last change:
Date: 1 year ago
Size: 6,076 bytes
 

Contents

Class file image Download
#!/usr/bin/env bash
#
# Cache file purging script.
# This script is part of ip2asn PHP library.
#
# @version    2020-10-05 06:31:00 UTC
# @author     Peter Kahl <https://github.com/peterkahl>
# @copyright  2015-2020 Peter Kahl
# @license    Apache License, Version 2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      <http://www.apache.org/licenses/LICENSE-2.0>
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

# Disable unicode
LC_ALL=C
LANG=C

TSTARTM="$(date +"%s.%N")"

# Cache directory
CACHEDIR="/srv/bgp"

# Cache time in seconds
CACHETIME="345600" # 4 days

# If a files has more than MAX_LINES, it will be tailed to $REDUCETO_LINES
MAX_LINES="500000"

REDUCETO_LINES="100000"

MODULENAME="ip2asn"

SUBNAME="purger"

# Whether to provide debug info?
# 0 ..... only errors
# 1 ..... medium
# 2 ..... every useless detail
LOG_LEVEL="2"

debugLog="${CACHEDIR}/${MODULENAME}_debug.log"

TLIMIT="$(($(date +"%s")-CACHETIME))"

# ====================================================================

function lineExists()
{
  # lineExists filename string
  cat "$1" | grep "$2" >/dev/null 2>&1 && \
    return 0 || \
    return 1
}

function milliStopwatch()
{
  local intval="$(echo "$(date +"%s.%N")-$1" | bc -l)"
  local seconds
  local ninechars
  IFS="." read -r seconds ninechars <<< "$intval"
  (( seconds < 1 )) && \
    printf %s "$(echo "$ninechars" | cut -b1-3 | sed 's/^00*//').$(echo "$ninechars" | cut -b4-5)ms" || \
    printf %s "${seconds}.$(echo "$ninechars" | cut -b1-2)s"
}

function sec2days()
{
  local secs="$1"
  if (( secs >= 86400 ))
  then
    printf %s "$((secs/86400))d"
  elif (( secs >= 3600 ))
    then
    printf %s "$((secs/3600))h"
  elif (( secs >= 60 ))
    then
    printf %s "$((secs/60))m"
  else
    printf %s "${secs}s"
  fi
}

function log_write()
{
  # log_write <string> <severity>
  local string="$1"
  local severity="$2"
  (( severity <= LOG_LEVEL )) && echo "$(date +"%Y-%m-%d %H:%M:%S") $MODULENAME/$SUBNAME[$BASHPID]: $string" >> $debugLog
}

function RandomString()
{
  printf %s "$(openssl rand -base64 13 | tr -cd "[0-9A-Za-z]")"
}

function get_lcount()
{
  printf %s "$(wc -l $1 | cut -d " " -f1)"
}

READABLECTM="$(sec2days "$CACHETIME")"

# ====================================================================

ver="4"
cachefile="${CACHEDIR}/${MODULENAME}_v${ver}_asdata.cache"

log_write ">>>> Purging file $cachefile ; CACHETIME=${READABLECTM}" "1"

randstr="$(RandomString)"
TEMPA="${CACHEDIR}/${MODULENAME}_tmp_${randstr}_A.tmp"
TEMPB="${CACHEDIR}/${MODULENAME}_tmp_${randstr}_B.tmp"

if [ -s $cachefile ]
then
  lines="$(get_lcount $cachefile)"
  log_write " File has $lines lines" "2"
fi

# ====================================================================
# Remove outdated lines
if [ -s $cachefile ]
then
  deleted="0"
  tstamp="$(head -n 1 $cachefile | cut -d "|" -f1)"
  age="$(($(date +"%s")-tstamp))"
  if (( age > CACHETIME ))
  then
    cp $cachefile $TEMPA
    while IFS='|' read -r f1 f2 f3 f4 f5 f6 f7 f8
    do
      if (( f1 > TLIMIT ))
      then
        echo "$f1|$f2|$f3|$f4|$f5|$f6|$f7|$f8" >> $TEMPB
      else
        deleted="$((deleted+1))"
      fi
    done < $TEMPA
    if [ -s $TEMPB ]; then
      chown www-data:www-data $TEMPB && chmod 0644 $TEMPB
      mv $TEMPB $cachefile
    fi
    rm -f $TEMPA
    (( deleted > 0 )) && log_write "-STALE: Deleted $deleted lines" "1"
  else
    log_write " STALE: Oldest record is $(sec2days "$age") old" "2"
  fi
else
  log_write " STALE: File not found or empty" "2"
fi
# ====================================================================

ver="6"
cachefile="${CACHEDIR}/${MODULENAME}_v${ver}_asdata.cache"

log_write ">>>> Purging file $cachefile ; CACHETIME=${READABLECTM}" "1"

randstr="$(RandomString)"
TEMPA="${CACHEDIR}/${MODULENAME}_tmp_${randstr}_A.tmp"
TEMPB="${CACHEDIR}/${MODULENAME}_tmp_${randstr}_B.tmp"

if [ -s $cachefile ]
then
  lines="$(get_lcount $cachefile)"
  log_write " File has $lines lines" "2"
fi

# ====================================================================
# Remove outdated lines
if [ -s $cachefile ]
then
  deleted="0"
  tstamp="$(head -n 1 $cachefile | cut -d "|" -f1)"
  age="$(($(date +"%s")-tstamp))"
  if (( age > CACHETIME ))
  then
    cp $cachefile $TEMPA
    while IFS='|' read -r f1 f2 f3 f4 f5 f6 f7 f8
    do
      if (( f1 > TLIMIT ))
      then
        echo "$f1|$f2|$f3|$f4|$f5|$f6|$f7|$f8" >> $TEMPB
      else
        deleted="$((deleted+1))"
      fi
    done < $TEMPA
    if [ -s $TEMPB ]; then
      chown www-data:www-data $TEMPB && chmod 0644 $TEMPB
      mv $TEMPB $cachefile
    fi
    rm -f $TEMPA
    (( deleted > 0 )) && log_write "-STALE: Deleted $deleted lines" "1"
  else
    log_write " STALE: Oldest record is $(sec2days "$age") old" "2"
  fi
else
  log_write " STALE: File not found or empty" "2"
fi
# ====================================================================

log_write ">>>> Purging files ${CACHEDIR}/${MODULENAME}_prefixes_v*.json ; CACHETIME=${READABLECTM}" "1"

if (( LOG_LEVEL == 2 ))
then
  totalcnt="$(find $CACHEDIR -name "${MODULENAME}_prefixes_v*.json" -type f | wc -l)"
  log_write " PREFIXES: Found $totalcnt files" "2"
fi

stale_count="$(find $CACHEDIR -name "${MODULENAME}_prefixes_v*.json" -mmin +"$((CACHETIME/60))" -type f | wc -l)"

if (( stale_count > 0 ))
then
  find $CACHEDIR -name "${MODULENAME}_prefixes_v*.json" -mmin +"$((CACHETIME/60))" -type f -delete && \
    log_write "-PREFIXES: Deleted $stale_count files" "1"
else
  log_write " PREFIXES: Nothing to delete" "2"
fi

# ====================================================================

log_write "Process completed in $(milliStopwatch "$TSTARTM")" "2"

exit 0
For more information send a message to info at phpclasses dot org.