git-backup.py (5588B) - raw

      1 #!/usr/bin/env python3
      2 # Copyright (C) 2019 Oscar Benedito
      3 #
      4 # This program is free software: you can redistribute it and/or modify
      5 # it under the terms of the GNU Affero General Public License as
      6 # published by the Free Software Foundation, either version 3 of the
      7 # License, or (at your option) any later version.
      8 #
      9 # This program is distributed in the hope that it will be useful,
     10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
     11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
     12 # GNU Affero General Public License for more details.
     13 #
     14 # You should have received a copy of the GNU Affero General Public License
     15 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
     16 
     17 import datetime
     18 import json
     19 import os
     20 import sys
     21 import subprocess
     22 import git
     23 import requests
     24 
     25 
     26 ENV_GITLAB_TOKEN = "GITLAB_TOKEN"
     27 ENV_GITHUB_TOKEN = "GITHUB_TOKEN"
     28 ENV_CUSTOM_REPOSITORIES_PATH = "CUSTOM_REPOSITORIES_PATH"
     29 ENV_TARGET_DIR = "TARGET_DIR"
     30 
     31 
     32 def backup_gitlab(token):
     33     def get_repositories_data_gitlab(url, page):
     34         headers = {"PRIVATE-TOKEN": token}
     35         response = requests.get(url + "&page=" + str(page), headers=headers, timeout=(10, 60))
     36         response.raise_for_status()
     37         return response.json()
     38 
     39     url = "https://gitlab.com/api/v4/projects?per_page=100&membership=true"
     40     page = 1
     41     repositories = get_repositories_data_gitlab(url, page)
     42 
     43     backup_data["sites"]["gitlab.com"] = []
     44     while len(repositories) != 0:
     45         for repository in repositories:
     46             clone_dir = "repositories/gitlab.com/" + repository["path_with_namespace"]
     47             print("gitlab.com/" + repository["path_with_namespace"])
     48             if os.path.isdir(clone_dir):
     49                 git.cmd.Git(clone_dir).fetch()
     50             else:
     51                 subprocess.run(["git", "clone", "--mirror", "--", repository["ssh_url_to_repo"], clone_dir], check=True)
     52             backup_data["sites"]["gitlab.com"].append(
     53                 {
     54                     "name": repository["name"],
     55                     "description": repository["description"],
     56                     "path": repository["path_with_namespace"],
     57                     "clone_url": repository["ssh_url_to_repo"],
     58                 }
     59             )
     60         page += 1
     61         repositories = get_repositories_data_gitlab(url, page)
     62 
     63 
     64 def backup_github(token):
     65     def get_repositories_data_github(url, token, page):
     66         headers = {"Authorization": "token " + token}
     67         response = requests.get(url + "?page=" + str(page), headers=headers, timeout=(10, 60))
     68         response.raise_for_status()
     69         return response.json()
     70 
     71     url = "https://api.github.com/user/repos"
     72     page = 1
     73     repositories = get_repositories_data_github(url, token, page)
     74 
     75     backup_data["sites"]["github.com"] = []
     76     while len(repositories) != 0:
     77         for repository in repositories:
     78             clone_dir = "repositories/github.com/" + repository["full_name"]
     79             print("github.com/" + repository["full_name"])
     80             if os.path.isdir(clone_dir):
     81                 git.cmd.Git(clone_dir).fetch()
     82             else:
     83                 subprocess.run(["git", "clone", "--mirror", "--", repository["ssh_url"], clone_dir], check=True)
     84             backup_data["sites"]["github.com"].append(
     85                 {
     86                     "name": repository["name"],
     87                     "description": repository["description"],
     88                     "path": repository["full_name"],
     89                     "clone_url": repository["ssh_url"],
     90                 }
     91             )
     92         page += 1
     93         repositories = get_repositories_data_github(url, token, page)
     94 
     95 
     96 def backup_custom_repositories(repositories):
     97     for repository in repositories:
     98         clone_dir = "repositories/" + repository["host"] + "/" + repository["path"]
     99         print(repository["host"] + "/" + repository["path"])
    100         if os.path.isdir(clone_dir):
    101             git.cmd.Git(clone_dir).fetch()
    102         else:
    103             subprocess.run(["git", "clone", "--mirror", "--", repository["clone_url"], clone_dir], check=True)
    104         if repository["host"] not in backup_data["sites"]:
    105             backup_data["sites"][repository["host"]] = []
    106         backup_data["sites"][repository["host"]].append(
    107             {
    108                 "name": repository["name"],
    109                 "description": repository["description"],
    110                 "path": repository["path"],
    111                 "clone_url": repository["clone_url"],
    112             }
    113         )
    114 
    115 
    116 def main():
    117     try:
    118         os.makedirs(os.environ.get(ENV_TARGET_DIR), exist_ok=True)
    119         os.chdir(os.environ.get(ENV_TARGET_DIR))
    120     except Exception as e:
    121         raise RuntimeError("Failed to create backup directory") from e
    122 
    123     backup_data["time"] = str(datetime.datetime.now())
    124     backup_data["sites"] = {}
    125 
    126     token = os.environ.get(ENV_GITLAB_TOKEN)
    127     if token:
    128         backup_gitlab(token)
    129 
    130     token = os.environ.get(ENV_GITHUB_TOKEN)
    131     if token:
    132         backup_github(token)
    133 
    134     custom_repositories_path = os.environ.get(ENV_CUSTOM_REPOSITORIES_PATH)
    135     if custom_repositories_path:
    136         try:
    137             with open(custom_repositories_path, "r") as f:
    138                 repositories = json.load(f)
    139             backup_custom_repositories(repositories)
    140         except FileNotFoundError:
    141             raise RuntimeError("Missing configured repository list: " + custom_repositories_path)
    142 
    143     with open("backup_data.json", "w", encoding="utf-8") as output_file:
    144         json.dump(backup_data, output_file, ensure_ascii=False)
    145         output_file.close()
    146 
    147 
    148 backup_data = {}
    149 main()