commit f82d853ef367545878c168b444f89e8a42e445eb
parent 6d4c87b85d8c6512a5adc7b6c7daf6742daac8cc
Author: Oscar Benedito <oscar@oscarbenedito.com>
Date: Tue, 9 Dec 2025 23:10:47 +0100
Use explicit paths instead of hard-coded ones
Diffstat:
| M | README.md | | | 33 | ++++++++++++++++----------------- |
| M | git-backup.py | | | 182 | +++++++++++++++++++++++++++++++++++++++++++++++++++---------------------------- |
2 files changed, 134 insertions(+), 81 deletions(-)
diff --git a/README.md b/README.md
@@ -13,33 +13,31 @@ install the libraries with the command:
pip3 install -r requirements.txt
```
-You will also need a GitLab and a GitHub access token and crete a file named
-`tokens.json` with the following content:
-
-```json
-{
- "gitlab.com":"<GitLab token>",
- "github.com":"<GitHub token>"
-}
-```
-
-Finally, you must upload your ssh key to GitLab or GitHub if you have any
-private or internal repositories.
-
## Running the program
-To run the program, execute the file `git-backup.py`:
+To run the program, execute the file `git-backup.py`, you must set the
+`TARGET_DIR` variable to the path where you want the script to store all the
+backed up repositories.
```
python3 git-backup.py
```
+### Adding you GitLab and GitHub repositories
+
+You will also need a GitLab and a GitHub access token and crete a file for each
+token, with the content being only the token. You must set environment variables
+`GITLAB_TOKEN_PATH` and `GITHUB_TOKEN_PATH` with the path of the file with each
+token.
+
+Finally, you must upload your ssh key to GitLab or GitHub if you have any
+private or internal repositories.
+
### Adding custom repositories
If you want to backup repositories from which you are not a member on the
currently supported hosts, or if you want to add repositories from other hosts,
-you can do so creating a file named `custom_directories.json` with the following
-structure:
+you can do so creating a file with the following structure:
```json
[
@@ -55,7 +53,8 @@ structure:
You can add more than one object to the array and it will backup all of the
repositories. Make sure that you upload your ssh key to the hosts in case it is
-needed for authentification.
+needed for authentification. Set the environment variable
+`CUSTOM_REPOSITORIES_PATH` to the path of the file.
## Future updates
diff --git a/git-backup.py b/git-backup.py
@@ -14,101 +14,155 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
-import os
-import requests
-import json
import datetime
+import json
+import os
+import sys
import git
+import requests
-def get_repositories_data_gitlab(url, page):
- response = requests.get(url + '&page=' + str(page))
- return response.json()
-
-
-def get_repositories_data_github(url, token, page):
- headers = {'Authorization': 'token ' + token}
- response = requests.get(url + '?page=' + str(page), headers=headers)
- return response.json()
+ENV_GITLAB_TOKEN_PATH = "GITLAB_TOKEN_PATH"
+ENV_GITHUB_TOKEN_PATH = "GITHUB_TOKEN_PATH"
+ENV_CUSTOM_REPOSITORIES_PATH = "CUSTOM_REPOSITORIES_PATH"
+ENV_TARGET_DIR = "TARGET_DIR"
-backup_data = {}
-backup_data['time'] = str(datetime.datetime.now())
-backup_data['sites'] = {}
+def read_file(env_var):
+ token_path = os.environ.get(env_var)
+ if not token_path:
+ return None
+ try:
+ with open(token_path, "r") as f:
+ return f.read().strip()
+ except FileNotFoundError:
+ print("Error: File {token_path} not found", file=sys.stderr)
+ return None
-with open('tokens.json', 'r') as tokens_file:
- tokens = json.load(tokens_file)
+def backup_gitlab(token):
+ def get_repositories_data_gitlab(url, page):
+ response = requests.get(url + "&page=" + str(page))
+ return response.json()
-# gitlab.com
-if 'gitlab.com' in tokens:
- url = 'https://gitlab.com/api/v4/projects?private_token=' + tokens['gitlab.com'] + '&per_page=100&membership=true'
+ url = "https://gitlab.com/api/v4/projects?private_token=" + token + "&per_page=100&membership=true"
page = 1
repositories = get_repositories_data_gitlab(url, page)
- backup_data['sites']['gitlab.com'] = []
+ backup_data["sites"]["gitlab.com"] = []
while len(repositories) != 0:
for repository in repositories:
- clone_dir = 'repositories/gitlab.com/' + repository['path_with_namespace']
- print('gitlab.com/' + repository['path_with_namespace'])
+ clone_dir = "repositories/gitlab.com/" + repository["path_with_namespace"]
+ print("gitlab.com/" + repository["path_with_namespace"])
if os.path.isdir(clone_dir):
git.cmd.Git(clone_dir).fetch()
else:
- os.system('git clone --mirror ' + repository['ssh_url_to_repo'] + ' ' + clone_dir)
- backup_data['sites']['gitlab.com'].append({
- 'name': repository['name'],
- 'description': repository['description'],
- 'path': repository['path_with_namespace'],
- 'ssh_url': repository['ssh_url_to_repo']
- })
+ os.system("git clone --mirror " + repository["ssh_url_to_repo"] + " " + clone_dir)
+ backup_data["sites"]["gitlab.com"].append(
+ {
+ "name": repository["name"],
+ "description": repository["description"],
+ "path": repository["path_with_namespace"],
+ "clone_url": repository["ssh_url_to_repo"],
+ }
+ )
page += 1
repositories = get_repositories_data_gitlab(url, page)
-# github.com
-if 'github.com' in tokens:
- url = 'https://api.github.com/user/repos'
+
+def backup_github(token):
+ def get_repositories_data_github(url, token, page):
+ headers = {"Authorization": "token " + token}
+ response = requests.get(url + "?page=" + str(page), headers=headers)
+ return response.json()
+
+ url = "https://api.github.com/user/repos"
page = 1
- repositories = get_repositories_data_github(url, tokens['github.com'], page)
+ repositories = get_repositories_data_github(url, token, page)
- backup_data['sites']['github.com'] = []
+ backup_data["sites"]["github.com"] = []
while len(repositories) != 0:
for repository in repositories:
- clone_dir = 'repositories/github.com/' + repository['full_name']
- print('github.com/' + repository['full_name'])
+ clone_dir = "repositories/github.com/" + repository["full_name"]
+ print("github.com/" + repository["full_name"])
if os.path.isdir(clone_dir):
git.cmd.Git(clone_dir).fetch()
else:
- os.system('git clone --mirror ' + repository['ssh_url'] + ' ' + clone_dir)
- backup_data['sites']['github.com'].append({
- 'name': repository['name'],
- 'description': repository['description'],
- 'path': repository['full_name'],
- 'ssh_url': repository['ssh_url']
- })
+ os.system("git clone --mirror " + repository["ssh_url"] + " " + clone_dir)
+ backup_data["sites"]["github.com"].append(
+ {
+ "name": repository["name"],
+ "description": repository["description"],
+ "path": repository["full_name"],
+ "clone_url": repository["ssh_url"],
+ }
+ )
page += 1
- repositories = get_repositories_data_github(url, tokens['github.com'], page)
+ repositories = get_repositories_data_github(url, token, page)
-# custom
-if os.path.exists('custom_directories.json'):
- with open('custom_directories.json', 'r') as custom_file:
- repositories = json.load(custom_file)
+def backup_custom_repositories(repositories):
for repository in repositories:
- clone_dir = 'repositories/' + repository['host'] + '/' + repository['path']
- print(repository['host'] + '/' + repository['path'])
+ clone_dir = "repositories/" + repository["host"] + "/" + repository["path"]
+ print(repository["host"] + "/" + repository["path"])
if os.path.isdir(clone_dir):
git.cmd.Git(clone_dir).fetch()
else:
- os.system('git clone --mirror ' + repository['ssh_url'] + ' ' + clone_dir)
- if repository['host'] not in backup_data['sites']:
- backup_data['sites'][repository['host']] = []
- backup_data['sites'][repository['host']].append({
- 'name': repository['name'],
- 'description': repository['description'],
- 'path': repository['path'],
- 'ssh_url': repository['ssh_url']
- })
-
-with open('backup_data.json', 'w', encoding='utf-8') as output_file:
- json.dump(backup_data, output_file, ensure_ascii=False)
- output_file.close()
+ os.system("git clone --mirror " + repository["clone_url"] + " " + clone_dir)
+ if repository["host"] not in backup_data["sites"]:
+ backup_data["sites"][repository["host"]] = []
+ backup_data["sites"][repository["host"]].append(
+ {
+ "name": repository["name"],
+ "description": repository["description"],
+ "path": repository["path"],
+ "clone_url": repository["clone_url"],
+ }
+ )
+
+
+def main():
+ try:
+ os.makedirs(os.environ.get(ENV_TARGET_DIR), exist_ok=True)
+ os.chdir(os.environ.get(ENV_TARGET_DIR))
+ except Exception as e:
+ print(f"Failed to create directory due to error: {e}", file=sys.stderr)
+ return
+
+ backup_data["time"] = str(datetime.datetime.now())
+ backup_data["sites"] = {}
+
+ token_path = os.environ.get(ENV_GITLAB_TOKEN_PATH)
+ if token_path:
+ try:
+ with open(token_path, "r") as f:
+ token = f.read().strip()
+ backup_gitlab(token)
+ except FileNotFoundError:
+ print("Error: File " + token_path + " not found", file=sys.stderr)
+
+ token_path = os.environ.get(ENV_GITHUB_TOKEN_PATH)
+ if token_path:
+ try:
+ with open(token_path, "r") as f:
+ token = f.read().strip()
+ backup_github(token)
+ except FileNotFoundError:
+ print("Error: File " + token_path + " not found", file=sys.stderr)
+
+ custom_repositories_path = os.environ.get(ENV_CUSTOM_REPOSITORIES_PATH)
+ if custom_repositories_path:
+ try:
+ with open(custom_repositories_path, "r") as f:
+ repositories = json.load(f)
+ backup_custom_repositories(repositories)
+ except FileNotFoundError:
+ print("Error: File " + custom_repositories_path + " not found", file=sys.stderr)
+
+ with open("backup_data.json", "w", encoding="utf-8") as output_file:
+ json.dump(backup_data, output_file, ensure_ascii=False)
+ output_file.close()
+
+
+backup_data = {}
+main()
