goaccess-run-daily.sh (3784B) - raw


      1 #!/bin/sh
      2 # Copyright (C) 2020 Oscar Benedito <oscar@oscarbenedito.com>
      3 #
      4 # This program is free software: you can redistribute it and/or modify
      5 # it under the terms of the GNU Affero General Public License as
      6 # published by the Free Software Foundation, either version 3 of the
      7 # License, or (at your option) any later version.
      8 #
      9 # This program is distributed in the hope that it will be useful,
     10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
     11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
     12 # GNU Affero General Public License for more details.
     13 #
     14 # You should have received a copy of the GNU Affero General Public License
     15 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
     16 
     17 # Runs daily to generate stats with GoAccess.
     18 
     19 # directory for html output
     20 [ -z "$OUT_DIR" ] && echo "Error: variable OUT_DIR not set." && exit 1
     21 # directory with all the GoAccess databases
     22 [ -z "$DB_DIR" ] && echo "Error: variable DB_DIR not set." && exit 1
     23 # name of log (the script appends strings ".1", ".2.gz" assuming that logrotate
     24 # is used with delaycompress)
     25 [ -z "$LOGS_PREFIX" ] && echo "Error: variable LOGS_PREFIX not set." && exit 1
     26 
     27 # Shortcut to run GoAccess.
     28 # Arguments:
     29 #   1. Log file
     30 #   2. Output file
     31 #   3. (Optional) Database location
     32 #   4. (Optional) Load from disk? Options: 0 (default), 1
     33 #   5. (Optional) Keep DB files? Options: 0 (default), 1
     34 run_goaccess() {
     35   if [ "$#" -ge 2 ]; then
     36     cmd="goaccess $1 -o $2"
     37     cmd="${cmd} --log-format=COMBINED --static-file=.js --static-file=.css --static-file=.ico --static-file=.svg --static-file=.png"
     38     if [ "$#" -ge 3 ]; then
     39       cmd="${cmd} --db-path=$3"
     40     fi
     41     if [ "$#" -ge 4 ]; then
     42       if [ "$4" -eq 1 ]; then
     43         cmd="${cmd} --restore"
     44       fi
     45     fi
     46     if [ "$#" -ge 5 ]; then
     47       if [ "$5" -eq 1 ]; then
     48         cmd="${cmd} --persist"
     49       fi
     50     fi
     51     eval $cmd
     52   else
     53     exit 1
     54   fi
     55 }
     56 
     57 # Runs GoAccess for on a time interval (year, month, etc.) respects the fact
     58 # that the last day of the week will also be in the following week (because logs
     59 # are created at an hour in the middle of the day, the boundary days are not
     60 # complete).
     61 # Runs GoAccess with a filtered accesses file and without a filter.
     62 run_goaccess_time_interval() {
     63   if [ "$#" -eq 2 ]; then
     64     db="$DB_DIR/$1"
     65     filtered=$(mktemp)
     66     filter_file "$LOGS_PREFIX.1" > "$filtered"
     67     if [ -d "$db" ]; then
     68       mkdir -p "$OUT_DIR/$1"
     69       run_goaccess "$filtered" "$OUT_DIR/$1/index.html" "$db" 1 1
     70     else
     71       mkdir -p "$db"
     72       mkdir -p "$OUT_DIR/$1"
     73       run_goaccess "$filtered" "$OUT_DIR/$1/index.html" "$db" 0 1
     74     fi
     75 
     76     if [ "$1" != "$2" ]; then
     77       db="$DB_DIR/$2"
     78       if [ -d "$db" ]; then
     79         mkdir -p "$OUT_DIR/$2"
     80         run_goaccess "$filtered" "$OUT_DIR/$2/index.html" "$db" 1 0
     81         rm -rf "$db"
     82       fi
     83     fi
     84     rm "$filtered"
     85   else
     86     exit 1
     87   fi
     88 }
     89 
     90 filter_file() {
     91   grep -v ".well-known/acme-challenge" "$1"
     92 }
     93 
     94 # Day
     95 tmp=$(mktemp)
     96 tmp2=$(mktemp)
     97 log2=$(mktemp)
     98 out_today="$OUT_DIR/$(date --date="yesterday" +"d/%Y/%m/%d")"
     99 
    100 cp "$LOGS_PREFIX.2.gz" "$log2.gz"
    101 gunzip -f "$log2.gz"
    102 
    103 mkdir -p "$out_today"
    104 cat "$log2" "$LOGS_PREFIX.1" > "$tmp"
    105 filter_file "$tmp" > "$tmp2"
    106 run_goaccess "$tmp2" "$out_today/index.html"
    107 rm "$tmp" "$tmp2" "$log2"
    108 
    109 # Week
    110 today="$(date +"w/%G/%V")"
    111 yesterday="$(date --date="yesterday" +"w/%G/%V")"
    112 run_goaccess_time_interval "$today" "$yesterday"
    113 
    114 # Month
    115 today="$(date +"m/%Y/%m")"
    116 yesterday="$(date --date="yesterday" +"m/%Y/%m")"
    117 run_goaccess_time_interval "$today" "$yesterday"
    118 
    119 # Year
    120 today="$(date +"y/%Y")"
    121 yesterday="$(date --date="yesterday" +"y/%Y")"
    122 run_goaccess_time_interval "$today" "$yesterday"
    123 
    124 # All time
    125 today="all"
    126 yesterday="all"
    127 run_goaccess_time_interval "$today" "$yesterday"