git-backup.py (5383B) - raw

      1 #!/usr/bin/env python3
      2 # Copyright (C) 2019 Oscar Benedito
      3 #
      4 # This program is free software: you can redistribute it and/or modify
      5 # it under the terms of the GNU Affero General Public License as
      6 # published by the Free Software Foundation, either version 3 of the
      7 # License, or (at your option) any later version.
      8 #
      9 # This program is distributed in the hope that it will be useful,
     10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
     11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
     12 # GNU Affero General Public License for more details.
     13 #
     14 # You should have received a copy of the GNU Affero General Public License
     15 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
     16 
     17 import datetime
     18 import json
     19 import os
     20 import sys
     21 import git
     22 import requests
     23 
     24 
     25 ENV_GITLAB_TOKEN = "GITLAB_TOKEN"
     26 ENV_GITHUB_TOKEN = "GITHUB_TOKEN"
     27 ENV_CUSTOM_REPOSITORIES_PATH = "CUSTOM_REPOSITORIES_PATH"
     28 ENV_TARGET_DIR = "TARGET_DIR"
     29 
     30 
     31 def backup_gitlab(token):
     32     def get_repositories_data_gitlab(url, page):
     33         response = requests.get(url + "&page=" + str(page))
     34         return response.json()
     35 
     36     url = "https://gitlab.com/api/v4/projects?private_token=" + token + "&per_page=100&membership=true"
     37     page = 1
     38     repositories = get_repositories_data_gitlab(url, page)
     39 
     40     backup_data["sites"]["gitlab.com"] = []
     41     while len(repositories) != 0:
     42         for repository in repositories:
     43             clone_dir = "repositories/gitlab.com/" + repository["path_with_namespace"]
     44             print("gitlab.com/" + repository["path_with_namespace"])
     45             if os.path.isdir(clone_dir):
     46                 git.cmd.Git(clone_dir).fetch()
     47             else:
     48                 os.system("git clone --mirror " + repository["ssh_url_to_repo"] + " " + clone_dir)
     49             backup_data["sites"]["gitlab.com"].append(
     50                 {
     51                     "name": repository["name"],
     52                     "description": repository["description"],
     53                     "path": repository["path_with_namespace"],
     54                     "clone_url": repository["ssh_url_to_repo"],
     55                 }
     56             )
     57         page += 1
     58         repositories = get_repositories_data_gitlab(url, page)
     59 
     60 
     61 def backup_github(token):
     62     def get_repositories_data_github(url, token, page):
     63         headers = {"Authorization": "token " + token}
     64         response = requests.get(url + "?page=" + str(page), headers=headers)
     65         return response.json()
     66 
     67     url = "https://api.github.com/user/repos"
     68     page = 1
     69     repositories = get_repositories_data_github(url, token, page)
     70 
     71     backup_data["sites"]["github.com"] = []
     72     while len(repositories) != 0:
     73         for repository in repositories:
     74             clone_dir = "repositories/github.com/" + repository["full_name"]
     75             print("github.com/" + repository["full_name"])
     76             if os.path.isdir(clone_dir):
     77                 git.cmd.Git(clone_dir).fetch()
     78             else:
     79                 os.system("git clone --mirror " + repository["ssh_url"] + " " + clone_dir)
     80             backup_data["sites"]["github.com"].append(
     81                 {
     82                     "name": repository["name"],
     83                     "description": repository["description"],
     84                     "path": repository["full_name"],
     85                     "clone_url": repository["ssh_url"],
     86                 }
     87             )
     88         page += 1
     89         repositories = get_repositories_data_github(url, token, page)
     90 
     91 
     92 def backup_custom_repositories(repositories):
     93     for repository in repositories:
     94         clone_dir = "repositories/" + repository["host"] + "/" + repository["path"]
     95         print(repository["host"] + "/" + repository["path"])
     96         if os.path.isdir(clone_dir):
     97             git.cmd.Git(clone_dir).fetch()
     98         else:
     99             os.system("git clone --mirror " + repository["clone_url"] + " " + clone_dir)
    100         if repository["host"] not in backup_data["sites"]:
    101             backup_data["sites"][repository["host"]] = []
    102         backup_data["sites"][repository["host"]].append(
    103             {
    104                 "name": repository["name"],
    105                 "description": repository["description"],
    106                 "path": repository["path"],
    107                 "clone_url": repository["clone_url"],
    108             }
    109         )
    110 
    111 
    112 def main():
    113     try:
    114         os.makedirs(os.environ.get(ENV_TARGET_DIR), exist_ok=True)
    115         os.chdir(os.environ.get(ENV_TARGET_DIR))
    116     except Exception as e:
    117         print(f"Failed to create directory due to error: {e}", file=sys.stderr)
    118         return
    119 
    120     backup_data["time"] = str(datetime.datetime.now())
    121     backup_data["sites"] = {}
    122 
    123     token = os.environ.get(ENV_GITLAB_TOKEN)
    124     if token:
    125         backup_gitlab(token)
    126 
    127     token = os.environ.get(ENV_GITHUB_TOKEN)
    128     if token:
    129         backup_github(token)
    130 
    131     custom_repositories_path = os.environ.get(ENV_CUSTOM_REPOSITORIES_PATH)
    132     if custom_repositories_path:
    133         try:
    134             with open(custom_repositories_path, "r") as f:
    135                 repositories = json.load(f)
    136             backup_custom_repositories(repositories)
    137         except FileNotFoundError:
    138             print("Error: File " + custom_repositories_path + " not found", file=sys.stderr)
    139 
    140     with open("backup_data.json", "w", encoding="utf-8") as output_file:
    141         json.dump(backup_data, output_file, ensure_ascii=False)
    142         output_file.close()
    143 
    144 
    145 backup_data = {}
    146 main()