commit 4712940985f430b4acb532c633013dae872055d5
parent a9335e4f5764ca60f0c67f7fa01fdaf65be65f1c
Author: oscarbenedito <oscar@obenedito.org>
Date: Sun, 16 Feb 2020 20:58:13 +0100
Merge git-backup
Diffstat:
4 files changed, 182 insertions(+), 0 deletions(-)
diff --git a/.gitignore b/.gitignore
@@ -0,0 +1,4 @@
+tokens.json
+repositories
+backup_data.json
+custom_directories.json
diff --git a/git-backup/README.md b/git-backup/README.md
@@ -0,0 +1,60 @@
+# Git Backup
+
+Very simple script to back up all the repositories the user is member of on GitLab and GitHub.
+
+## Prerequisites
+
+In order to run the program, you will need `python3` and the python libraries specified in the `requirements.txt` file. If you have `pip3` installed, you can install the libraries with the command:
+
+```
+pip3 install -r requirements.txt
+```
+
+You will also need a GitLab and a GitHub access token and crete a file named `tokens.json` with the following content:
+```json
+{
+ "gitlab.com":"<GitLab token>",
+ "github.com":"<GitHub token>"
+}
+```
+
+Finally, you must upload your ssh key to GitLab or GitHub if you have any private or internal repositories.
+
+## Running the program
+
+To run the program, execute the file `git-backup.py`:
+
+```
+python3 git-backup.py
+```
+
+### Adding custom repositories
+
+If you want to backup repositories from which you are not a member on the currently supported hosts, or if you want to add repositories from other hosts, you can do so creating a file named `custom_directories.json` with the following structure:
+
+```json
+[
+ {
+ "name":"<Repository name - for backup information>",
+ "description":"<Repository description - for backup information>",
+ "path":"<Repository path where the backup will be saved>",
+ "ssh_url":"<Repository url>",
+ "host":"<Repository host - for backup information and establishing saving directory>"
+ }
+]
+```
+
+You can add more than one object to the array and it will backup all of the repositories. Make sure that you upload your ssh key to the hosts in case it is needed for authentification.
+
+## Future updates
+
+ - [ ] More information on the output file or the terminal: how many new repositories were added, how many have been deleted on the hosting services.
+ - [ ] Manage what happens if history has changed in the hosting service.
+
+## License
+
+The program is licensed under the GPL v3. License is available [here](https://www.gnu.org/licenses/gpl-3.0.html).
+
+## Author
+
+ - **Oscar Benedito** - oscar@obenedito.org
diff --git a/git-backup/git-backup.py b/git-backup/git-backup.py
@@ -0,0 +1,116 @@
+#!/usr/bin/env python3
+# Copyright (C) 2019 Oscar Benedito
+#
+# This file is part of Git Backup.
+#
+# Git Backup is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Git Backup is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Git Backup. If not, see <https://www.gnu.org/licenses/>.
+
+import os
+import requests
+import json
+import datetime
+import git
+
+
+def get_repositories_data_gitlab(url, page):
+ response = requests.get(url + '&page=' + str(page))
+ return response.json()
+
+
+def get_repositories_data_github(url, token, page):
+ headers = {'Authorization': 'token ' + token}
+ response = requests.get(url + '?page=' + str(page), headers=headers)
+ return response.json()
+
+
+backup_data = {}
+backup_data['time'] = str(datetime.datetime.now())
+backup_data['sites'] = {}
+
+with open('tokens.json', 'r') as tokens_file:
+ tokens = json.load(tokens_file)
+
+
+# gitlab.com
+if 'gitlab.com' in tokens:
+ url = 'https://gitlab.com/api/v4/projects?private_token=' + tokens['gitlab.com'] + '&per_page=100&membership=true'
+ page = 1
+ repositories = get_repositories_data_gitlab(url, page)
+
+ backup_data['sites']['gitlab.com'] = []
+ while len(repositories) != 0:
+ for repository in repositories:
+ clone_dir = 'repositories/gitlab.com/' + repository['path_with_namespace']
+ print('gitlab.com/' + repository['path_with_namespace'])
+ if os.path.isdir(clone_dir):
+ git.cmd.Git(clone_dir).fetch()
+ else:
+ os.system('git clone --mirror ' + repository['ssh_url_to_repo'] + ' ' + clone_dir)
+ backup_data['sites']['gitlab.com'].append({
+ 'name': repository['name'],
+ 'description': repository['description'],
+ 'path': repository['path_with_namespace'],
+ 'ssh_url': repository['ssh_url_to_repo']
+ })
+ page += 1
+ repositories = get_repositories_data_gitlab(url, page)
+
+# github.com
+if 'github.com' in tokens:
+ url = 'https://api.github.com/user/repos'
+ page = 1
+ repositories = get_repositories_data_github(url, tokens['github.com'], page)
+
+ backup_data['sites']['github.com'] = []
+ while len(repositories) != 0:
+ for repository in repositories:
+ clone_dir = 'repositories/github.com/' + repository['full_name']
+ print('github.com/' + repository['full_name'])
+ if os.path.isdir(clone_dir):
+ git.cmd.Git(clone_dir).fetch()
+ else:
+ os.system('git clone --mirror ' + repository['ssh_url'] + ' ' + clone_dir)
+ backup_data['sites']['github.com'].append({
+ 'name': repository['name'],
+ 'description': repository['description'],
+ 'path': repository['full_name'],
+ 'ssh_url': repository['ssh_url']
+ })
+ page += 1
+ repositories = get_repositories_data_github(url, tokens['github.com'], page)
+
+# custom
+if os.path.exists("custom_directories.json"):
+ custom_file = open('custom_directories.json', 'r')
+ repositories = json.load(custom_file)
+ custom_file.close()
+ for repository in repositories:
+ clone_dir = 'repositories/' + repository['host'] + '/' + repository['path']
+ print(repository['host'] + '/' + repository['path'])
+ if os.path.isdir(clone_dir):
+ git.cmd.Git(clone_dir).fetch()
+ else:
+ os.system('git clone --mirror ' + repository['ssh_url'] + ' ' + clone_dir)
+ if repository['host'] not in backup_data['sites']:
+ backup_data['sites'][repository['host']] = []
+ backup_data['sites'][repository['host']].append({
+ 'name': repository['name'],
+ 'description': repository['description'],
+ 'path': repository['path'],
+ 'ssh_url': repository['ssh_url']
+ })
+
+with open('backup_data.json', 'w', encoding='utf-8') as output_file:
+ json.dump(backup_data, output_file, ensure_ascii=False)
+ output_file.close()
diff --git a/git-backup/requirements.txt b/git-backup/requirements.txt
@@ -0,0 +1,2 @@
+requests
+GitPython
