goaccess-run-daily.sh (3186B) - raw

      1 #!/bin/sh
      2 # Copyright (C) 2020 Oscar Benedito <oscar@oscarbenedito.com>
      3 # License: Affero General Public License version 3 or later
      4 
      5 # Runs daily to generate stats with GoAccess.
      6 
      7 # directory for html output
      8 [ -z "$OUT_DIR" ] && echo "Error: variable OUT_DIR not set." && exit 1
      9 # directory with all the GoAccess databases
     10 [ -z "$DB_DIR" ] && echo "Error: variable DB_DIR not set." && exit 1
     11 # name of log (the script appends strings ".1", ".2.gz" assuming that logrotate
     12 # is used with delaycompress)
     13 [ -z "$LOGS_PREFIX" ] && echo "Error: variable LOGS_PREFIX not set." && exit 1
     14 
     15 # Shortcut to run GoAccess.
     16 # Arguments:
     17 #   1. Log file
     18 #   2. Output file
     19 #   3. (Optional) Database location
     20 #   4. (Optional) Load from disk? Options: 0 (default), 1
     21 #   5. (Optional) Keep DB files? Options: 0 (default), 1
     22 run_goaccess() {
     23   if [ "$#" -ge 2 ]; then
     24     cmd="goaccess $1 -o $2"
     25     cmd="${cmd} --log-format=COMBINED --static-file=.js --static-file=.css --static-file=.ico --static-file=.svg --static-file=.png"
     26     if [ "$#" -ge 3 ]; then
     27       cmd="${cmd} --db-path=$3"
     28     fi
     29     if [ "$#" -ge 4 ]; then
     30       if [ "$4" -eq 1 ]; then
     31         cmd="${cmd} --restore"
     32       fi
     33     fi
     34     if [ "$#" -ge 5 ]; then
     35       if [ "$5" -eq 1 ]; then
     36         cmd="${cmd} --persist"
     37       fi
     38     fi
     39     eval $cmd
     40   else
     41     exit 1
     42   fi
     43 }
     44 
     45 # Runs GoAccess for on a time interval (year, month, etc.) respects the fact
     46 # that the last day of the week will also be in the following week (because logs
     47 # are created at an hour in the middle of the day, the boundary days are not
     48 # complete).
     49 # Runs GoAccess with a filtered accesses file and without a filter.
     50 run_goaccess_time_interval() {
     51   if [ "$#" -eq 2 ]; then
     52     db="$DB_DIR/$1"
     53     filtered=$(mktemp)
     54     filter_file "$LOGS_PREFIX.1" > "$filtered"
     55     if [ -d "$db" ]; then
     56       mkdir -p "$OUT_DIR/$1"
     57       run_goaccess "$filtered" "$OUT_DIR/$1/index.html" "$db" 1 1
     58     else
     59       mkdir -p "$db"
     60       mkdir -p "$OUT_DIR/$1"
     61       run_goaccess "$filtered" "$OUT_DIR/$1/index.html" "$db" 0 1
     62     fi
     63 
     64     if [ "$1" != "$2" ]; then
     65       db="$DB_DIR/$2"
     66       if [ -d "$db" ]; then
     67         mkdir -p "$OUT_DIR/$2"
     68         run_goaccess "$filtered" "$OUT_DIR/$2/index.html" "$db" 1 0
     69         rm -rf "$db"
     70       fi
     71     fi
     72     rm "$filtered"
     73   else
     74     exit 1
     75   fi
     76 }
     77 
     78 filter_file() {
     79   grep -v ".well-known/acme-challenge" "$1"
     80 }
     81 
     82 # Day
     83 tmp=$(mktemp)
     84 tmp2=$(mktemp)
     85 log2=$(mktemp)
     86 out_today="$OUT_DIR/$(date --date="yesterday" +"d/%Y/%m/%d")"
     87 
     88 cp "$LOGS_PREFIX.2.gz" "$log2.gz"
     89 gunzip -f "$log2.gz"
     90 
     91 mkdir -p "$out_today"
     92 cat "$log2" "$LOGS_PREFIX.1" > "$tmp"
     93 filter_file "$tmp" > "$tmp2"
     94 run_goaccess "$tmp2" "$out_today/index.html"
     95 rm "$tmp" "$tmp2" "$log2"
     96 
     97 # Week
     98 today="$(date +"w/%G/%V")"
     99 yesterday="$(date --date="yesterday" +"w/%G/%V")"
    100 run_goaccess_time_interval "$today" "$yesterday"
    101 
    102 # Month
    103 today="$(date +"m/%Y/%m")"
    104 yesterday="$(date --date="yesterday" +"m/%Y/%m")"
    105 run_goaccess_time_interval "$today" "$yesterday"
    106 
    107 # Year
    108 today="$(date +"y/%Y")"
    109 yesterday="$(date --date="yesterday" +"y/%Y")"
    110 run_goaccess_time_interval "$today" "$yesterday"
    111 
    112 # All time
    113 today="all"
    114 yesterday="all"
    115 run_goaccess_time_interval "$today" "$yesterday"