commit 94fdb58598991f5bcdf3d02befbbcb1d6994c52e parent 8e81ae4dcfc23c2bc623a268c42b83eef0c6fe30 Author: Oscar Benedito <oscar@oscarbenedito.com> Date: Mon, 17 Aug 2020 11:10:14 +0200 Change organization Now all scripts are self contained (no need for individual READMEs) and are all on the root folder. Diffstat:
| A | .gitignore | | | 4 | ++++ |
| M | README.md | | | 5 | +++-- |
| A | atenea-updates-notifications.py | | | 105 | +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ |
| D | atenea-updates-notifications/.gitignore | | | 1 | - |
| D | atenea-updates-notifications/README.md | | | 36 | ------------------------------------ |
| D | atenea-updates-notifications/atenea-updates-notifications.py | | | 88 | ------------------------------------------------------------------------------- |
| A | check-changes-website.sh | | | 37 | +++++++++++++++++++++++++++++++++++++ |
| A | davup.sh | | | 66 | ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ |
| A | deploy-website.sh | | | 41 | +++++++++++++++++++++++++++++++++++++++++ |
| A | goaccess-run-daily.sh | | | 123 | +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ |
| D | goaccess-stats/README.md | | | 15 | --------------- |
| D | goaccess-stats/goaccess-run-daily.sh | | | 121 | ------------------------------------------------------------------------------- |
| D | goaccess-stats/stats-index-generator.py | | | 168 | ------------------------------------------------------------------------------- |
| A | hex-base64-translator.py | | | 123 | +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ |
| D | hex-base64-translator/README.md | | | 29 | ----------------------------- |
| D | hex-base64-translator/hex-base64-translator.py | | | 107 | ------------------------------------------------------------------------------- |
| A | login-notify.sh | | | 32 | ++++++++++++++++++++++++++++++++ |
| D | miscellanea/.gitignore | | | 1 | - |
| D | miscellanea/check-changes-website.sh | | | 38 | -------------------------------------- |
| D | miscellanea/davup.sh | | | 66 | ------------------------------------------------------------------ |
| D | miscellanea/login-notify.sh | | | 33 | --------------------------------- |
| A | post-hugo-script.py | | | 46 | ++++++++++++++++++++++++++++++++++++++++++++++ |
| A | stats-index-generator.py | | | 170 | +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ |
| A | website-backup.sh | | | 46 | ++++++++++++++++++++++++++++++++++++++++++++++ |
| D | website-backup/.gitignore | | | 2 | -- |
| D | website-backup/website-backup.sh | | | 46 | ---------------------------------------------- |
| D | website-deployment/.gitignore | | | 2 | -- |
| D | website-deployment/deploy-website.sh | | | 41 | ----------------------------------------- |
| D | website-deployment/post-hugo-script.py | | | 46 | ---------------------------------------------- |
29 files changed, 796 insertions(+), 842 deletions(-)
diff --git a/.gitignore b/.gitignore @@ -0,0 +1,4 @@ +aun_config.json +gotify_token.txt +post_hugo_script.json +urls.txt diff --git a/README.md b/README.md @@ -1,4 +1,4 @@ -# Utilities +# Scripts This repository gathers some of the scripts I use. @@ -12,6 +12,7 @@ Public License version 3 or later (available [here][agpl]). - **Oscar Benedito** - oscar@oscarbenedito.com *Note*: For some scripts, I have had help from other people. Co-authors are -clearly stated on those files. +stated on those files. + [agpl]: <https://www.gnu.org/licenses/agpl-3.0.html> "GNU Affero General Public License v3.0" diff --git a/atenea-updates-notifications.py b/atenea-updates-notifications.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python3 +# Atenea Updates Notifications +# Copyright (C) 2020 Oscar Benedito <oscar@oscarbenedito.com> +# Copyright (C) 2020 Ernesto Lanchares <e.lancha98@gmail.com> +# Copyright (C) 2020 Ferran López <flg@tuta.io> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <https://www.gnu.org/licenses/>. + +# Get notified when new documents are uploaded to Atenea (an instance of +# Moodle). Makes use of a Gotify server. +# +# The script assumes there is a file name "aun_config.json" with the +# configuration. Example: +# +# { +# "notification_domain": "<gotify-domain>", +# "time_interval": 120, +# "api_token": "<moodle-api-token>", +# "notification_token": "<gotify-token>", +# "course_ids": { +# "56145": "GD", +# "56152": "EDPS" +# } +# } + + +import os +import requests +import json +import time + + +with open(os.path.join(os.path.dirname(os.path.realpath(__file__)), 'aun_config.json'), 'r') as f: + CONFIG = json.load(f) + + +NOTIFICATION_DOMAIN = CONFIG['notification_domain'] +TIME_INTERVAL = CONFIG['time_interval'] +COURSE_IDS = CONFIG['course_ids'] +API_TOKEN = CONFIG['api_token'] +NOTIFICATION_TOKEN = CONFIG['notification_token'] + + +def get_updates(id): + parameters = { + 'wstoken': API_TOKEN, + 'moodlewsrestformat': 'json', + 'wsfunction': 'core_course_get_updates_since', + 'courseid': id, + 'since': int(time.time()) - (TIME_INTERVAL + 20) + } + response = requests.get('https://atenea.upc.edu/webservice/rest/server.php', params=parameters) + return response.json()['instances'] + + +def get_course_docs(id): + parameters = { + 'wstoken': API_TOKEN, + 'moodlewsrestformat': 'json', + 'wsfunction': 'core_course_get_contents', + 'courseid': id, + } + return requests.get('https://atenea.upc.edu/webservice/rest/server.php', params=parameters) + + +def find_document(docs, doc_id): + for module in docs: + for doc in module['modules']: + if doc['id'] == doc_id: + return doc + + +def send_notification(doc, course_name): + if doc['modname'] == 'resource': + message = 'URL: ' + doc['contents'][0]['fileurl'] + '&token=' + API_TOKEN + else: + message = doc['modplural'] + + data = { + 'title': course_name + ': ' + doc['name'], + 'message': message, + 'priority': 5 + } + requests.post('https://' + NOTIFICATION_DOMAIN + '/message?token=' + NOTIFICATION_TOKEN, data = data) + +for id, course_name in COURSE_IDS.items(): + updates = get_updates(id) + + if updates != []: + course_docs = get_course_docs(id) + + for update in updates: + doc = find_document(course_docs.json(), update['id']) + send_notification(doc, course_name) diff --git a/atenea-updates-notifications/.gitignore b/atenea-updates-notifications/.gitignore @@ -1 +0,0 @@ -config.json diff --git a/atenea-updates-notifications/README.md b/atenea-updates-notifications/README.md @@ -1,36 +0,0 @@ -# Atenea Updates Notifications - -Very simple Python script to get notified when new documents are uploaded to -Atenea (an instance of Moodle). Makes use of a Gotify server. - -## Example `config.json` - -The `.json` file with the configuration looks like the following: - -```json -{ - "notification_domain": "<gotify-domain>", - "time_interval": 120, - "api_token": "<moodle-api-token>", - "notification_token": "<gotify-token>", - "course_ids": { - "56145": "GD", - "56152": "EDPS" - } -} -``` - -## License - -The program is licensed under the GNU Affero General Public License version 3 or -later (available [here][agpl]). - -## Authors - -Alphabetically by last name. - -- **Oscar Benedito** - oscar@oscarbenedito.com -- **Ernesto Lanchares** - e.lancha98@gmail.com -- **Ferran López** - flg@tuta.io - -[agpl]: <https://www.gnu.org/licenses/agpl-3.0.html> "The GNU General Public License v3.0" diff --git a/atenea-updates-notifications/atenea-updates-notifications.py b/atenea-updates-notifications/atenea-updates-notifications.py @@ -1,88 +0,0 @@ -#!/usr/bin/env python3 -# Atenea Updates Notifications -# Copyright (C) 2020 Oscar Benedito <oscar@oscarbenedito.com> -# Copyright (C) 2020 Ernesto Lanchares <e.lancha98@gmail.com> -# Copyright (C) 2020 Ferran López <flg@tuta.io> -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see <https://www.gnu.org/licenses/>. - - -import os -import requests -import json -import time - - -with open(os.path.join(os.path.dirname(os.path.realpath(__file__)), 'config.json'), 'r') as f: - CONFIG = json.load(f) - - -NOTIFICATION_DOMAIN = CONFIG['notification_domain'] -TIME_INTERVAL = CONFIG['time_interval'] -COURSE_IDS = CONFIG['course_ids'] -API_TOKEN = CONFIG['api_token'] -NOTIFICATION_TOKEN = CONFIG['notification_token'] - - -def get_updates(id): - parameters = { - 'wstoken': API_TOKEN, - 'moodlewsrestformat': 'json', - 'wsfunction': 'core_course_get_updates_since', - 'courseid': id, - 'since': int(time.time()) - (TIME_INTERVAL + 20) - } - response = requests.get('https://atenea.upc.edu/webservice/rest/server.php', params=parameters) - return response.json()['instances'] - - -def get_course_docs(id): - parameters = { - 'wstoken': API_TOKEN, - 'moodlewsrestformat': 'json', - 'wsfunction': 'core_course_get_contents', - 'courseid': id, - } - return requests.get('https://atenea.upc.edu/webservice/rest/server.php', params=parameters) - - -def find_document(docs, doc_id): - for module in docs: - for doc in module['modules']: - if doc['id'] == doc_id: - return doc - - -def send_notification(doc, course_name): - if doc['modname'] == 'resource': - message = 'URL: ' + doc['contents'][0]['fileurl'] + '&token=' + API_TOKEN - else: - message = doc['modplural'] - - data = { - 'title': course_name + ': ' + doc['name'], - 'message': message, - 'priority': 5 - } - requests.post('https://' + NOTIFICATION_DOMAIN + '/message?token=' + NOTIFICATION_TOKEN, data = data) - -for id, course_name in COURSE_IDS.items(): - updates = get_updates(id) - - if updates != []: - course_docs = get_course_docs(id) - - for update in updates: - doc = find_document(course_docs.json(), update['id']) - send_notification(doc, course_name) diff --git a/check-changes-website.sh b/check-changes-website.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env sh +# Copyright (C) 2020 Oscar Benedito <oscar@oscarbenedito.com> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <https://www.gnu.org/licenses/>. + +# Script that notifies through Gotify when a website has changed. + +GOTIFY_DOMAIN="gotify.oscarbenedito.com" +API_TOKEN="$(cat "$(dirname "$(realpath "$0")")/gotify_token.txt")" + +check_and_send_message() { + newhash="$(curl "$URL" 2>/dev/null | sha256sum | cut -f 1 -d " ")" + [ "$HASH" != "$newhash" ] && \ + curl -X POST "https://$GOTIFY_DOMAIN/message?token=$API_TOKEN" \ + -F "title=$TITLE" \ + -F "message=$URL" \ + -F "priority=5" \ + >/dev/null 2>&1 +} + + +HASH="<hash>" +URL="<url>" +TITLE="<title>" + +check_and_send_message diff --git a/davup.sh b/davup.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env sh +# DAVup +# Copyright (C) 2020 Oscar Benedito <oscar@oscarbenedito.com> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <https://www.gnu.org/licenses/>. + +# Back up calendars and contacts from a DAV server (CalDAV and CardDAV). + +domain="<DAV server domain" # example: https://dav.mailbox.org +user="<username>" +pass="<password>" + +get_cal() { + curl -s -X "PROPFIND" -u "${user}:${pass}" -H "Content-Type: text/xml" -H "Depth: 1" \ + --data "<propfind xmlns='DAV:'><prop><calendar-data xmlns='urn:ietf:params:xml:ns:caldav'/></prop></propfind>" \ + "${domain}/${resource}" +} + +get_card() { + curl -s -X "PROPFIND" -u "${user}:${pass}" -H "Content-Type: text/xml" -H "Depth: 1" \ + --data "<propfind xmlns='DAV:'><prop><address-data xmlns=\"urn:ietf:params:xml:ns:carddav\"/></prop></propfind>" \ + "${domain}/${resource}" +} + +process_cal() { + intro="BEGIN:VCALENDAR" + pre="BEGIN:VEVENT" + post="END:VEVENT" + step="0" + while read -r line; do + case $step in + 0) echo "${line%}" | grep -q "${intro}" && step="1" && echo "${intro}" ;; + 1) echo "${line%}" && [ "${line%}" = "${post}" ] && step="2" ;; + 2) [ "${line%}" = "${pre}" ] && step="1" && echo "${line%}" ;; + esac + done + echo "END:VCALENDAR" +} + +process_card() { + pre="BEGIN:VCARD" + post="END:VCARD" + step="1" + while read -r line; do + case $step in + 1) echo "${line%}" | grep -q "${pre}" && step="2" && echo "${pre}" ;; + 2) echo "${line%}" | grep -q "${post}" && step="1" && echo "${post}" \ + || echo "${line%}" ;; + esac + done +} + +# examples +resource="caldav/mycal" && get_cal | process_cal > calendar.ics +resource="carddav/mycard" && get_card | process_card > contacts.vcf diff --git a/deploy-website.sh b/deploy-website.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env sh +# Copyright (C) 2020 Oscar Benedito <oscar@oscarbenedito.com> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <https://www.gnu.org/licenses/>. + +# Script to deploy a website built with Hugo. + +HUGO_PATH="/root/oscarbenedito.com" +WEB_PATH="/srv/oscarbenedito.com" +GOTIFY_DOMAIN="gotify.oscarbenedito.com" +FILE_DIR="$(dirname "$(realpath "$0")")" + +git -C $HUGO_PATH fetch origin master +git -C $HUGO_PATH reset --hard origin/master +git -C $HUGO_PATH verify-commit master || exit 1 +rm -rf $HUGO_PATH/public +rm -rf $HUGO_PATH/resources +hugo -s $HUGO_PATH --gc +$FILE_DIR/post-hugo-script.py "$FILE_DIR/post_hugo_script.json" +rsync --perms --recursive --checksum --delete "$HUGO_PATH/public/" "$WEB_PATH" + +API_TOKEN="$(cat "$FILE_DIR/gotify_token.txt")" +TITLE="Web update triggered" +MESSAGE="Git hooks triggered an update of the website." + +curl -X POST "https://$GOTIFY_DOMAIN/message?token=$API_TOKEN" \ + -F "title=$TITLE" \ + -F "message=$MESSAGE" \ + -F "priority=5" \ + >/dev/null 2>&1 diff --git a/goaccess-run-daily.sh b/goaccess-run-daily.sh @@ -0,0 +1,123 @@ +#!/usr/bin/env sh +# Copyright (C) 2020 Oscar Benedito <oscar@oscarbenedito.com> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <https://www.gnu.org/licenses/>. + +# Runs daily to generate stats with GoAccess. + +OUT_DIR="/srv/stats" +DB_DIR="/srv/goaccess-db" +LOGS_PREFIX="/var/log/apache2/a-website.log" + +# Shortcut to run GoAccess. +# Arguments: +# 1. Log file +# 2. Output file +# 3. (Optional) Database location +# 4. (Optional) Load from disk? Options: 0 (default), 1 +# 5. (Optional) Keep DB files? Options: 0 (default), 1 +run_goaccess() { + if [ "$#" -ge 2 ]; then + cmd="goaccess $1 -o $2" + cmd="${cmd} --log-format=COMBINED --static-file=.js --static-file=.css --static-file=.ico --static-file=.svg --static-file=.png" + if [ "$#" -ge 3 ]; then + cmd="${cmd} --db-path=$3" + fi + if [ "$#" -ge 4 ]; then + if [ "$4" -eq 1 ]; then + cmd="${cmd} --restore" + fi + fi + if [ "$#" -ge 5 ]; then + if [ "$5" -eq 1 ]; then + cmd="${cmd} --persist" + fi + fi + eval $cmd + else + exit 1 + fi +} + +# Runs GoAccess for on a time interval (year, month, etc.) respects the fact +# that the last day of the week will also be in the following week (because logs +# are created at an hour in the middle of the day, the boundary days are not +# complete). +# Runs GoAccess with a filtered accesses file and without a filter. +run_goaccess_time_interval() { + if [ "$#" -eq 2 ]; then + DB="$DB_DIR/$1" + FILTERED_FILE=$(mktemp) + filter_file "$LOGS_PREFIX.1" > "$FILTERED_FILE" + if [ -d "$DB" ]; then + mkdir -p "$OUT_DIR/$1" + run_goaccess "$FILTERED_FILE" "$OUT_DIR/$1/index.html" "$DB" 1 1 + else + mkdir -p "$DB" + mkdir -p "$OUT_DIR/$1" + run_goaccess "$FILTERED_FILE" "$OUT_DIR/$1/index.html" "$DB" 0 1 + fi + + if [ "$1" != "$2" ]; then + DB="$DB_DIR/$2" + if [ -d "$DB" ]; then + mkdir -p "$OUT_DIR/$2" + run_goaccess "$FILTERED_FILE" "$OUT_DIR/$2/index.html" "$DB" 1 0 + rm -rf "$DB" + fi + fi + rm "$FILTERED_FILE" + else + exit 1 + fi +} + +filter_file() { + grep -v ".well-known/acme-challenge" "$1" +} + +# Day +TMP_FILE=$(mktemp) +TMP_FILE2=$(mktemp) +LOGS_2=$(mktemp) +OUT_DIR_TODAY="$OUT_DIR/$(date --date="yesterday" +"d/%Y/%m/%d")" + +cp "$LOGS_PREFIX.2.gz" "$LOGS_2.gz" +gunzip -f "$LOGS_2.gz" + +mkdir -p "$OUT_DIR_TODAY" +cat "$LOGS_2" "$LOGS_PREFIX.1" > "$TMP_FILE" +filter_file "$TMP_FILE" > "$TMP_FILE2" +run_goaccess "$TMP_FILE2" "$OUT_DIR_TODAY/index.html" +rm "$TMP_FILE" "$TMP_FILE2" "$LOGS_2" + +# Week +TODAY="$(date +"w/%G/%V")" +YESTERDAY="$(date --date="yesterday" +"w/%G/%V")" +run_goaccess_time_interval "$TODAY" "$YESTERDAY" + +# Month +TODAY="$(date +"m/%Y/%m")" +YESTERDAY="$(date --date="yesterday" +"m/%Y/%m")" +run_goaccess_time_interval "$TODAY" "$YESTERDAY" + +# Year +TODAY="$(date +"y/%Y")" +YESTERDAY="$(date --date="yesterday" +"y/%Y")" +run_goaccess_time_interval "$TODAY" "$YESTERDAY" + +# All time +TODAY="all" +YESTERDAY="all" +run_goaccess_time_interval "$TODAY" "$YESTERDAY" diff --git a/goaccess-stats/README.md b/goaccess-stats/README.md @@ -1,15 +0,0 @@ -# GoAccess stats - -This scripts create different outputs for GoAccess to check statistics on a -given website. - -## License - -The program is licensed under the GNU Affero General Public License version 3 or -later (available [here][agpl]). - -## Author - -- **Oscar Benedito** - oscar@oscarbenedito.com - -[agpl]: <https://www.gnu.org/licenses/agpl-3.0.html> "GNU Affero General Public License v3.0" diff --git a/goaccess-stats/goaccess-run-daily.sh b/goaccess-stats/goaccess-run-daily.sh @@ -1,121 +0,0 @@ -#!/usr/bin/env sh -# Copyright (C) 2020 Oscar Benedito -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see <https://www.gnu.org/licenses/>. - -OUT_DIR="/srv/stats" -DB_DIR="/srv/goaccess-db" -LOGS_PREFIX="/var/log/apache2/a-website.log" - -# Shortcut to run GoAccess. -# Arguments: -# 1. Log file -# 2. Output file -# 3. (Optional) Database location -# 4. (Optional) Load from disk? Options: 0 (default), 1 -# 5. (Optional) Keep DB files? Options: 0 (default), 1 -run_goaccess() { - if [ "$#" -ge 2 ]; then - cmd="goaccess $1 -o $2" - cmd="${cmd} --log-format=COMBINED --static-file=.js --static-file=.css --static-file=.ico --static-file=.svg --static-file=.png" - if [ "$#" -ge 3 ]; then - cmd="${cmd} --db-path=$3" - fi - if [ "$#" -ge 4 ]; then - if [ "$4" -eq 1 ]; then - cmd="${cmd} --restore" - fi - fi - if [ "$#" -ge 5 ]; then - if [ "$5" -eq 1 ]; then - cmd="${cmd} --persist" - fi - fi - eval $cmd - else - exit 1 - fi -} - -# Runs GoAccess for on a time interval (year, month, etc.) respects the fact -# that the last day of the week will also be in the following week (because logs -# are created at an hour in the middle of the day, the boundary days are not -# complete). -# Runs GoAccess with a filtered accesses file and without a filter. -run_goaccess_time_interval() { - if [ "$#" -eq 2 ]; then - DB="$DB_DIR/$1" - FILTERED_FILE=$(mktemp) - filter_file "$LOGS_PREFIX.1" > "$FILTERED_FILE" - if [ -d "$DB" ]; then - mkdir -p "$OUT_DIR/$1" - run_goaccess "$FILTERED_FILE" "$OUT_DIR/$1/index.html" "$DB" 1 1 - else - mkdir -p "$DB" - mkdir -p "$OUT_DIR/$1" - run_goaccess "$FILTERED_FILE" "$OUT_DIR/$1/index.html" "$DB" 0 1 - fi - - if [ "$1" != "$2" ]; then - DB="$DB_DIR/$2" - if [ -d "$DB" ]; then - mkdir -p "$OUT_DIR/$2" - run_goaccess "$FILTERED_FILE" "$OUT_DIR/$2/index.html" "$DB" 1 0 - rm -rf "$DB" - fi - fi - rm "$FILTERED_FILE" - else - exit 1 - fi -} - -filter_file() { - grep -v ".well-known/acme-challenge" "$1" -} - -# Day -TMP_FILE=$(mktemp) -TMP_FILE2=$(mktemp) -LOGS_2=$(mktemp) -OUT_DIR_TODAY="$OUT_DIR/$(date --date="yesterday" +"d/%Y/%m/%d")" - -cp "$LOGS_PREFIX.2.gz" "$LOGS_2.gz" -gunzip -f "$LOGS_2.gz" - -mkdir -p "$OUT_DIR_TODAY" -cat "$LOGS_2" "$LOGS_PREFIX.1" > "$TMP_FILE" -filter_file "$TMP_FILE" > "$TMP_FILE2" -run_goaccess "$TMP_FILE2" "$OUT_DIR_TODAY/index.html" -rm "$TMP_FILE" "$TMP_FILE2" "$LOGS_2" - -# Week -TODAY="$(date +"w/%G/%V")" -YESTERDAY="$(date --date="yesterday" +"w/%G/%V")" -run_goaccess_time_interval "$TODAY" "$YESTERDAY" - -# Month -TODAY="$(date +"m/%Y/%m")" -YESTERDAY="$(date --date="yesterday" +"m/%Y/%m")" -run_goaccess_time_interval "$TODAY" "$YESTERDAY" - -# Year -TODAY="$(date +"y/%Y")" -YESTERDAY="$(date --date="yesterday" +"y/%Y")" -run_goaccess_time_interval "$TODAY" "$YESTERDAY" - -# All time -TODAY="all" -YESTERDAY="all" -run_goaccess_time_interval "$TODAY" "$YESTERDAY" diff --git a/goaccess-stats/stats-index-generator.py b/goaccess-stats/stats-index-generator.py @@ -1,168 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (C) 2020 Oscar Benedito -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see <https://www.gnu.org/licenses/>. - -import datetime -import os.path - -OUTPUT_DIR = '/srv/stats' - -def has_info(link): - return os.path.isfile(OUTPUT_DIR + link + '/index.html') - -output = """<!DOCTYPE html> -<html lang="en"> - <head> - <meta charset="utf-8"> - <meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=1,user-scalable=no"> - <meta name="description" content="Oscar Benedito's website stats"> - <meta name="author" content="Oscar Benedito"> - <title>Website stats</title> - - <link rel="icon" href="/favicon.min.svg"> - <link rel="shortcut icon" href="/favicon.ico"> - <style> -body { - font-family: 'sans-serif'; - margin: 3em; -} - -a.no-format { - color: inherit; - text-decoration: none; -} - -a.hover:hover { - text-decoration: underline; -} - -h1, -h2 { - text-align: center; -} - -div.year { - margin: 5em auto; -} - -div.year-cal { - display: flex; - flex-flow: row wrap; - justify-content: flex-start; - max-width: 1024px; - margin: 0 auto; -} - -div.month { - width: 17em; - margin: 0; - padding: 1em 0; - flex: 1; -} - -div.month ul { - width: 17em; - margin: 0 auto; - padding: 0em 1em 1em 1em; -} - -div.month li { - float: left; - display: block; - width: 12.5%; - text-align: center; - list-style-type: none; -} - -div.month a li:hover { - border-radius: 5px; - background-color: #1abc9c; - color: #ecf0f1 !important; -} - -div.month li.today { - border-radius: 5px; - background-color: #5759D4; - color: #ecf0f1; -} - -div.month li.week { - font-weight: 900; - color: #e67e22; -} - </style> - </head> - - <body> -""" - -output += '<h1><a class="no-format hover" href="/all">See stats of all time</a></h1>' - -for year in [2020]: - link = '/y/' + str(year) - if has_info(link): - output += '<div class="year"><h1><a class="no-format hover" href="' + link + '">' + str(year) + '</a></h1><div class="year-cal">' - else: - output += '<div class="year"><h1>' + str(year) + '</h1><div class="year-cal">' - - for month in range(1, 13): - date = datetime.datetime(year, month, 1) - link = date.strftime("/m/%Y/%m") - if has_info(link): - output += '<div class="month"><h2><a class="no-format hover" href="' + link + '">' + datetime.datetime(year, month, 1).strftime("%B") + '</a></h2><ul>' - else: - output += '<div class="month"><h2>' + datetime.datetime(year, month, 1).strftime("%B") + '</h2><ul>' - - if date.isocalendar()[2] != 1: - date = datetime.datetime(year, month, 1) - isoweek = date.strftime("%V") - link = date.strftime("/w/%G/%V") - if has_info(link): - output += '<a class="no-format" href="' + link + '"><li class="week">' + isoweek + '</li></a>' - else: - output += '<li class="week">' + isoweek + '</li>' - - for i in range(date.isocalendar()[2] - 1): - output += '<li style="opacity: 0;">-</li>' - - for day in range(1, 32): - try: - date = datetime.datetime(year, month, day) - except: - break - - if date.isocalendar()[2] == 1: - isoweek = date.strftime("%V") - link = date.strftime("/w/%G/%V") - if has_info(link): - output += '<a class="no-format" href="' + link + '"><li class="week">' + isoweek + '</li></a>' - else: - output += '<li class="week">' + isoweek + '</li>' - - today = ' class="today"' if datetime.datetime.today().date() == date.date() else '' - link = date.strftime("/d/%Y/%m/%d") - if has_info(link): - output += '<a class="no-format" href="' + link + '"><li' + today + '>' + str(day) + '</li></a>' - else: - output += '<li' + today + '>' + str(day) + '</li>' - - output += '</ul></div>' - - output += '</div></div>' - -output += '</body></html>\n' - -with open(OUTPUT_DIR + '/index.html', 'wt') as f: - f.write(output) diff --git a/hex-base64-translator.py b/hex-base64-translator.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python3 +# Copyright (C) 2020 Oscar Benedito <oscar@oscarbenedito.com> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <https://www.gnu.org/licenses/>. + +# This script simply translates hexadecimal strings into Base64 strings (by +# converting them into bits and then reading the bits as Base64) and the inverse +# process. +# +# The goal of this program is make a password containing only characters in the +# Base64 alphabet and then being able to split the secret between different +# parties using ssss (<http://point-at-infinity.org/ssss/>) with the hexadecimal +# option. With this, an attacker can't get any advantage by discarding unvalid +# answers, since they are all valid (when running the program normally, you can +# get "binary" secrets or "ASCII" secrets). +# +# All this trouble is due to the fact that I am not sure if there is a way for +# an attacker with some shares of the secret to avoid making a brute-force +# attack by knowing the implementation of ssss and anticipate binary (and +# therefore invalid) results. + +import sys +from getpass import getpass + + +def char_to_bits(c): + n = ord(c) + if n >= 65 and n <= 90: + return bin(n - 65)[2:].zfill(6) + elif n >= 97 and n <= 122: + return bin(n - 71)[2:].zfill(6) + elif n >= 48 and n <= 57: + return bin(n + 4)[2:].zfill(6) + elif (c == '+'): + return bin(62)[2:].zfill(6) + elif (c == '/'): + return bin(63)[2:].zfill(6) + else: + sys.exit('Error, ' + c + ' is not a Base64 character.', file=sys.stderr) + + +def bits_to_char(s): + n = int(s, 2) + if n < 26: + return chr(n + 65) + elif n < 52: + return chr(n + 71) + elif n < 62: + return chr(n - 4) + elif n == 62: + return '+' + elif n == 63: + return '/' + else: + sys.exit('Error, ' + s + ' (' + str(n) + ') is not a binary number lower than 64.', file=sys.stderr) + + +def base64_to_hex(s): + if len(s) % 2: + print('WARNING: Number of Base64 characters is not multiple of 2. Adding zeros to string.', file=sys.stderr) + s = 'A' + s + + ret = '' + carry = '' + while len(s) > 0: + cs = s[:2] + bs = char_to_bits(s[0]) + char_to_bits(s[1]) + ret += hex(int(bs, 2))[2:].zfill(3) + s = s[2:] + + return ret + + +def hex_to_base64(s): + if len(s) % 3: + print('WARNING: Number of hexadecimal values is not a multiple of 3. Adding zeros to string.', file=sys.stderr) + s = '0'*(3 - (len(s) % 3)) + s + + ret = '' + while len(s) > 0: + bs = bin(int(s[:3], 16))[2:].zfill(12) + ret += bits_to_char(bs[:6]) + ret += bits_to_char(bs[6:]) + s = s[3:] + + return ret + + +if __name__ == '__main__': + if len(sys.argv) != 2 or (sys.argv[1] != 'base64-to-hex' and sys.argv[1] != 'hex-to-base64'): + sys.exit('Usage: ' + sys.argv[0] + ' base64-to-hex | hex-to-base64.') + + if sys.argv[1] == 'base64-to-hex': + inp = getpass(prompt = 'Base64 secret: ') + print('') + out = base64_to_hex(inp) + print('-'*80) + print('Secret in hexadecimal:', out) + print('-'*80) + + elif sys.argv[1] == 'hex-to-base64': + inp = getpass(prompt = 'Hexadecimal secret: ') + print('') + out = hex_to_base64(inp) + print('-'*80) + print('Secret in Base64:', out) + print('-'*80) + if inp[0] == '0' and len(inp) % 2 == 0: + out = hex_to_base64(inp[1:]) + print('-'*80) + print('Due to SSSS having an output with an even number of characters, your secret could be:', out) + print('-'*80) diff --git a/hex-base64-translator/README.md b/hex-base64-translator/README.md @@ -1,29 +0,0 @@ -# Hex and Base64 translator - -This script simply translates hexadecimal strings into Base64 strings (by -converting them into bits and then reading the bits as Base64) and the inverse -process. - -The goal of this program is make a password containing only characters in the -Base64 alphabet and then being able to split the secret between different -parties using [ssss][ssss] (with the hexadecimal -option). With this, an attacker can't get any advantage by discarding unvalid -answers, since they are all valid (when running the program normally, you can -get "binary" secrets or "ASCII" secrets). - -All this trouble is due to the fact that I am not sure if there is a way for an -attacker with some shares of the secret to avoid making a brute-force attack by -knowing the implementation of ssss and anticipate binary (and therefore invalid) -results. - -## License - -The program is licensed under the GNU Affero General Public License version 3 or -later (available [here][agpl]). - -## Author - -- **Oscar Benedito** - oscar@oscarbenedito.com - -[ssss]: <http://point-at-infinity.org/ssss/> "Shamir's Secret Sharing Scheme" -[agpl]: <https://www.gnu.org/licenses/agpl-3.0.html> "GNU Affero General Public License v3.0" diff --git a/hex-base64-translator/hex-base64-translator.py b/hex-base64-translator/hex-base64-translator.py @@ -1,107 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (C) 2020 Oscar Benedito -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see <https://www.gnu.org/licenses/>. - -import sys -from getpass import getpass - - -def char_to_bits(c): - n = ord(c) - if n >= 65 and n <= 90: - return bin(n - 65)[2:].zfill(6) - elif n >= 97 and n <= 122: - return bin(n - 71)[2:].zfill(6) - elif n >= 48 and n <= 57: - return bin(n + 4)[2:].zfill(6) - elif (c == '+'): - return bin(62)[2:].zfill(6) - elif (c == '/'): - return bin(63)[2:].zfill(6) - else: - sys.exit('Error, ' + c + ' is not a Base64 character.', file=sys.stderr) - - -def bits_to_char(s): - n = int(s, 2) - if n < 26: - return chr(n + 65) - elif n < 52: - return chr(n + 71) - elif n < 62: - return chr(n - 4) - elif n == 62: - return '+' - elif n == 63: - return '/' - else: - sys.exit('Error, ' + s + ' (' + str(n) + ') is not a binary number lower than 64.', file=sys.stderr) - - -def base64_to_hex(s): - if len(s) % 2: - print('WARNING: Number of Base64 characters is not multiple of 2. Adding zeros to string.', file=sys.stderr) - s = 'A' + s - - ret = '' - carry = '' - while len(s) > 0: - cs = s[:2] - bs = char_to_bits(s[0]) + char_to_bits(s[1]) - ret += hex(int(bs, 2))[2:].zfill(3) - s = s[2:] - - return ret - - -def hex_to_base64(s): - if len(s) % 3: - print('WARNING: Number of hexadecimal values is not a multiple of 3. Adding zeros to string.', file=sys.stderr) - s = '0'*(3 - (len(s) % 3)) + s - - ret = '' - while len(s) > 0: - bs = bin(int(s[:3], 16))[2:].zfill(12) - ret += bits_to_char(bs[:6]) - ret += bits_to_char(bs[6:]) - s = s[3:] - - return ret - - -if __name__ == '__main__': - if len(sys.argv) != 2 or (sys.argv[1] != 'base64-to-hex' and sys.argv[1] != 'hex-to-base64'): - sys.exit('Usage: ' + sys.argv[0] + ' base64-to-hex | hex-to-base64.') - - if sys.argv[1] == 'base64-to-hex': - inp = getpass(prompt = 'Base64 secret: ') - print('') - out = base64_to_hex(inp) - print('-'*80) - print('Secret in hexadecimal:', out) - print('-'*80) - - elif sys.argv[1] == 'hex-to-base64': - inp = getpass(prompt = 'Hexadecimal secret: ') - print('') - out = hex_to_base64(inp) - print('-'*80) - print('Secret in Base64:', out) - print('-'*80) - if inp[0] == '0' and len(inp) % 2 == 0: - out = hex_to_base64(inp[1:]) - print('-'*80) - print('Due to SSSS having an output with an even number of characters, your secret could be:', out) - print('-'*80) diff --git a/login-notify.sh b/login-notify.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env sh +# Copyright (C) 2020 Oscar Benedito <oscar@oscarbenedito.com> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <https://www.gnu.org/licenses/>. + +# Script that notifies Gotify when someone logs in through SSH to a computer/server. + +GOTIFY_DOMAIN="gotify.oscarbenedito.com" +API_TOKEN="$(cat "$(dirname "$(realpath "$0")")/gotify_token.txt")" + +if [ "$PAM_TYPE" != "close_session" ]; then + TITLE="SSH login: ${PAM_USER}@$(hostname)" + MESSAGE="IP: ${PAM_RHOST} +Date: $(TZ='Europe/Madrid' date)" + + curl -X POST "https://$GOTIFY_DOMAIN/message?token=$API_TOKEN" \ + -F "title=$TITLE" \ + -F "message=$MESSAGE" \ + -F "priority=5" \ + >/dev/null 2>&1 +fi diff --git a/miscellanea/.gitignore b/miscellanea/.gitignore @@ -1 +0,0 @@ -login_api_token.txt diff --git a/miscellanea/check-changes-website.sh b/miscellanea/check-changes-website.sh @@ -1,38 +0,0 @@ -#!/bin/sh -# Copyright (C) 2020 Oscar Benedito -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see <https://www.gnu.org/licenses/>. - -# Script that notifies through Gotify when a website has changed. - - -GOTIFY_DOMAIN="gotify.oscarbenedito.com" -API_TOKEN="<redacted>" - -check_and_send_message() { - newhash="$(curl "$URL" 2>/dev/null | sha256sum | cut -f 1 -d " ")" - [ "$HASH" != "$newhash" ] && \ - curl -X POST "https://$GOTIFY_DOMAIN/message?token=$API_TOKEN" \ - -F "title=$TITLE" \ - -F "message=$URL" \ - -F "priority=5" \ - >/dev/null 2>&1 -} - - -HASH="<hash>" -URL="<url>" -TITLE="<title>" - -check_and_send_message diff --git a/miscellanea/davup.sh b/miscellanea/davup.sh @@ -1,66 +0,0 @@ -#!/usr/bin/env sh -# DAVup -# Copyright (C) 2020 Oscar Benedito -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see <https://www.gnu.org/licenses/>. - -# Back up calendars and contacts from a DAV server (CalDAV and CardDAV). - -domain="<DAV server domain" # example: https://dav.mailbox.org -user="<username>" -pass="<password>" - -get_cal() { - curl -s -X "PROPFIND" -u "${user}:${pass}" -H "Content-Type: text/xml" -H "Depth: 1" \ - --data "<propfind xmlns='DAV:'><prop><calendar-data xmlns='urn:ietf:params:xml:ns:caldav'/></prop></propfind>" \ - "${domain}/${resource}" -} - -get_card() { - curl -s -X "PROPFIND" -u "${user}:${pass}" -H "Content-Type: text/xml" -H "Depth: 1" \ - --data "<propfind xmlns='DAV:'><prop><address-data xmlns=\"urn:ietf:params:xml:ns:carddav\"/></prop></propfind>" \ - "${domain}/${resource}" -} - -process_cal() { - intro="BEGIN:VCALENDAR" - pre="BEGIN:VEVENT" - post="END:VEVENT" - step="0" - while read -r line; do - case $step in - 0) echo "${line%}" | grep -q "${intro}" && step="1" && echo "${intro}" ;; - 1) echo "${line%}" && [ "${line%}" = "${post}" ] && step="2" ;; - 2) [ "${line%}" = "${pre}" ] && step="1" && echo "${line%}" ;; - esac - done - echo "END:VCALENDAR" -} - -process_card() { - pre="BEGIN:VCARD" - post="END:VCARD" - step="1" - while read -r line; do - case $step in - 1) echo "${line%}" | grep -q "${pre}" && step="2" && echo "${pre}" ;; - 2) echo "${line%}" | grep -q "${post}" && step="1" && echo "${post}" \ - || echo "${line%}" ;; - esac - done -} - -# examples -resource="caldav/mycal" && get_cal | process_cal > calendar.ics -resource="carddav/mycard" && get_card | process_card > contacts.vcf diff --git a/miscellanea/login-notify.sh b/miscellanea/login-notify.sh @@ -1,33 +0,0 @@ -#!/usr/bin/env sh -# Copyright (C) 2020 Oscar Benedito -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see <https://www.gnu.org/licenses/>. - -# Script that notifies Gotify when someone logs in through SSH to a computer/server. - -GOTIFY_DOMAIN="gotify.oscarbenedito.com" - -API_TOKEN="$(cat "$(dirname "$(realpath "$0")")/login_api_token.txt")" - -if [ "$PAM_TYPE" != "close_session" ]; then - TITLE="SSH login: ${PAM_USER}@$(hostname)" - MESSAGE="IP: ${PAM_RHOST} -Date: $(TZ='Europe/Madrid' date)" - - curl -X POST "https://$GOTIFY_DOMAIN/message?token=$API_TOKEN" \ - -F "title=$TITLE" \ - -F "message=$MESSAGE" \ - -F "priority=5" \ - >/dev/null 2>&1 -fi diff --git a/post-hugo-script.py b/post-hugo-script.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 +# Copyright (C) 2020 Oscar Benedito <oscar@oscarbenedito.com> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <https://www.gnu.org/licenses/>. + +import os +import sys +import re +import json +import shutil + +HUGO_OUT_DIR = '/root/oscarbenedito.com/public' + +if len(sys.argv) != 2: + print("Usage:\n", sys.argv[0], "file.json") + exit(1) + +with open(sys.argv[1], 'r') as f: + data = json.load(f) + +for dir in data['directories']: + shutil.rmtree(os.path.join(HUGO_OUT_DIR, dir)) + +for file in data['files']: + os.remove(os.path.join(HUGO_OUT_DIR, file)) + +with open(os.path.join(HUGO_OUT_DIR, 'sitemap.xml'), 'r') as f: + sitemap = f.read() + +for line in data['sitemap']: + block = '\n[ \t]*<url>\n[ \t]*<loc>https://oscarbenedito\.com' + line + '</loc>\n(?:[ \t]*<lastmod>.*</lastmod>\n)?[ \t]*</url>\n[ \t]*' + sitemap = re.sub(block, '', sitemap) + +with open(os.path.join(HUGO_OUT_DIR, 'sitemap.xml'), 'w') as f: + f.write(sitemap) diff --git a/stats-index-generator.py b/stats-index-generator.py @@ -0,0 +1,170 @@ +#!/usr/bin/env python3 +# Copyright (C) 2020 Oscar Benedito <oscar@oscarbenedito.com> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <https://www.gnu.org/licenses/>. + +# Generate index for GoAccess stats. + +import datetime +import os.path + +OUTPUT_DIR = '/srv/stats' + +def has_info(link): + return os.path.isfile(OUTPUT_DIR + link + '/index.html') + +output = """<!DOCTYPE html> +<html lang="en"> + <head> + <meta charset="utf-8"> + <meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=1,user-scalable=no"> + <meta name="description" content="Oscar Benedito's website stats"> + <meta name="author" content="Oscar Benedito"> + <title>Website stats</title> + + <link rel="icon" href="/favicon.min.svg"> + <link rel="shortcut icon" href="/favicon.ico"> + <style> +body { + font-family: 'sans-serif'; + margin: 3em; +} + +a.no-format { + color: inherit; + text-decoration: none; +} + +a.hover:hover { + text-decoration: underline; +} + +h1, +h2 { + text-align: center; +} + +div.year { + margin: 5em auto; +} + +div.year-cal { + display: flex; + flex-flow: row wrap; + justify-content: flex-start; + max-width: 1024px; + margin: 0 auto; +} + +div.month { + width: 17em; + margin: 0; + padding: 1em 0; + flex: 1; +} + +div.month ul { + width: 17em; + margin: 0 auto; + padding: 0em 1em 1em 1em; +} + +div.month li { + float: left; + display: block; + width: 12.5%; + text-align: center; + list-style-type: none; +} + +div.month a li:hover { + border-radius: 5px; + background-color: #1abc9c; + color: #ecf0f1 !important; +} + +div.month li.today { + border-radius: 5px; + background-color: #5759D4; + color: #ecf0f1; +} + +div.month li.week { + font-weight: 900; + color: #e67e22; +} + </style> + </head> + + <body> +""" + +output += '<h1><a class="no-format hover" href="/all">See stats of all time</a></h1>' + +for year in [2020]: + link = '/y/' + str(year) + if has_info(link): + output += '<div class="year"><h1><a class="no-format hover" href="' + link + '">' + str(year) + '</a></h1><div class="year-cal">' + else: + output += '<div class="year"><h1>' + str(year) + '</h1><div class="year-cal">' + + for month in range(1, 13): + date = datetime.datetime(year, month, 1) + link = date.strftime("/m/%Y/%m") + if has_info(link): + output += '<div class="month"><h2><a class="no-format hover" href="' + link + '">' + datetime.datetime(year, month, 1).strftime("%B") + '</a></h2><ul>' + else: + output += '<div class="month"><h2>' + datetime.datetime(year, month, 1).strftime("%B") + '</h2><ul>' + + if date.isocalendar()[2] != 1: + date = datetime.datetime(year, month, 1) + isoweek = date.strftime("%V") + link = date.strftime("/w/%G/%V") + if has_info(link): + output += '<a class="no-format" href="' + link + '"><li class="week">' + isoweek + '</li></a>' + else: + output += '<li class="week">' + isoweek + '</li>' + + for i in range(date.isocalendar()[2] - 1): + output += '<li style="opacity: 0;">-</li>' + + for day in range(1, 32): + try: + date = datetime.datetime(year, month, day) + except: + break + + if date.isocalendar()[2] == 1: + isoweek = date.strftime("%V") + link = date.strftime("/w/%G/%V") + if has_info(link): + output += '<a class="no-format" href="' + link + '"><li class="week">' + isoweek + '</li></a>' + else: + output += '<li class="week">' + isoweek + '</li>' + + today = ' class="today"' if datetime.datetime.today().date() == date.date() else '' + link = date.strftime("/d/%Y/%m/%d") + if has_info(link): + output += '<a class="no-format" href="' + link + '"><li' + today + '>' + str(day) + '</li></a>' + else: + output += '<li' + today + '>' + str(day) + '</li>' + + output += '</ul></div>' + + output += '</div></div>' + +output += '</body></html>\n' + +with open(OUTPUT_DIR + '/index.html', 'wt') as f: + f.write(output) diff --git a/website-backup.sh b/website-backup.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env sh +# Copyright (C) 2020 Oscar Benedito <oscar@oscarbenedito.com> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <https://www.gnu.org/licenses/>. + +# Script to backup content on the Internet. It gets a list of URLs and +# destination files and puts each document in the corresponding file, +# adding a date to the filename. + +FILE_DIR="$(dirname "$(realpath "$0")")" +GOTIFY_DOMAIN="gotify.oscarbenedito.com" +API_TOKEN="$(cat "$FILE_DIR/gotify_token.txt")" +URL_FILE="$FILE_DIR/urls.txt" +BACKUP_PATH="$HOME/backups" + +save() { wget --quiet --output-document "$OUTPUT" "$URL" ; } + +error_message () { + TITLE="Website backup error" + MESSAGE="Error backing up $OUTPUT" + + curl -X POST "https://$GOTIFY_DOMAIN/message?token=$API_TOKEN" \ + -F "title=$TITLE" \ + -F "message=$MESSAGE" \ + -F "priority=5" \ + >/dev/null 2>&1 +} + +while IFS= read -r line +do + mkdir -p "$BACKUP_PATH/${line#* }" + OUTPUT="$BACKUP_PATH/${line#* }/$(date +"%Y-%m-%d")-${line#* }" + URL="${line% *}" + save || error_message +done < "$URL_FILE" diff --git a/website-backup/.gitignore b/website-backup/.gitignore @@ -1,2 +0,0 @@ -website_api_token.txt -urls.txt diff --git a/website-backup/website-backup.sh b/website-backup/website-backup.sh @@ -1,46 +0,0 @@ -#!/usr/bin/env sh -# Copyright (C) 2020 Oscar Benedito -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see <https://www.gnu.org/licenses/>. - -# Script to backup content on the Internet. It gets a list of URLs and -# destination files and puts each document in the corresponding file, -# adding a date to the filename. - -FILE_DIR="$(dirname "$(realpath "$0")")" -GOTIFY_DOMAIN="gotify.oscarbenedito.com" -API_TOKEN="$(cat "$FILE_DIR/website_api_token.txt")" -URL_FILE="$FILE_DIR/urls.txt" -BACKUP_PATH="$HOME/backups" - -save() { wget --quiet --output-document "$OUTPUT" "$URL" ; } - -error_message () { - TITLE="Website backup error" - MESSAGE="Error backing up $OUTPUT" - - curl -X POST "https://$GOTIFY_DOMAIN/message?token=$API_TOKEN" \ - -F "title=$TITLE" \ - -F "message=$MESSAGE" \ - -F "priority=5" \ - >/dev/null 2>&1 -} - -while IFS= read -r line -do - mkdir -p "$BACKUP_PATH/${line#* }" - OUTPUT="$BACKUP_PATH/${line#* }/$(date +"%Y-%m-%d")-${line#* }" - URL="${line% *}" - save || error_message -done < "$URL_FILE" diff --git a/website-deployment/.gitignore b/website-deployment/.gitignore @@ -1,2 +0,0 @@ -website_api_token.txt -post_hugo_script.json diff --git a/website-deployment/deploy-website.sh b/website-deployment/deploy-website.sh @@ -1,41 +0,0 @@ -#!/usr/bin/env sh -# Copyright (C) 2020 Oscar Benedito -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see <https://www.gnu.org/licenses/>. - -# Script to deploy a website built with Hugo. - -HUGO_PATH="/root/oscarbenedito.com" -WEB_PATH="/srv/oscarbenedito.com" -GOTIFY_DOMAIN="gotify.oscarbenedito.com" -FILE_DIR="$(dirname "$(realpath "$0")")" - -git -C $HUGO_PATH fetch origin master -git -C $HUGO_PATH reset --hard origin/master -git -C $HUGO_PATH verify-commit master || exit 1 -rm -rf $HUGO_PATH/public -rm -rf $HUGO_PATH/resources -hugo -s $HUGO_PATH --gc -$FILE_DIR/post-hugo-script.py "$FILE_DIR/post_hugo_script.json" -rsync --perms --recursive --checksum --delete "$HUGO_PATH/public/" "$WEB_PATH" - -API_TOKEN="$(cat "$FILE_DIR/website_api_token.txt")" -TITLE="Web update triggered" -MESSAGE="Git hooks triggered an update of the website." - -curl -X POST "https://$GOTIFY_DOMAIN/message?token=$API_TOKEN" \ - -F "title=$TITLE" \ - -F "message=$MESSAGE" \ - -F "priority=5" \ - >/dev/null 2>&1 diff --git a/website-deployment/post-hugo-script.py b/website-deployment/post-hugo-script.py @@ -1,46 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (C) 2020 Oscar Benedito -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see <https://www.gnu.org/licenses/>. - -import os -import sys -import re -import json -import shutil - -HUGO_OUT_DIR = '/root/oscarbenedito.com/public' - -if len(sys.argv) != 2: - print("Usage:\n", sys.argv[0], "file.json") - exit(1) - -with open(sys.argv[1], 'r') as f: - data = json.load(f) - -for dir in data['directories']: - shutil.rmtree(os.path.join(HUGO_OUT_DIR, dir)) - -for file in data['files']: - os.remove(os.path.join(HUGO_OUT_DIR, file)) - -with open(os.path.join(HUGO_OUT_DIR, 'sitemap.xml'), 'r') as f: - sitemap = f.read() - -for line in data['sitemap']: - block = '\n[ \t]*<url>\n[ \t]*<loc>https://oscarbenedito\.com' + line + '</loc>\n(?:[ \t]*<lastmod>.*</lastmod>\n)?[ \t]*</url>\n[ \t]*' - sitemap = re.sub(block, '', sitemap) - -with open(os.path.join(HUGO_OUT_DIR, 'sitemap.xml'), 'w') as f: - f.write(sitemap)