mirror of
https://github.com/YuzuZensai/Zen-Sync.git
synced 2026-01-05 20:31:04 +00:00
✨ feat: first release
This commit is contained in:
184
.gitignore
vendored
Normal file
184
.gitignore
vendored
Normal file
@@ -0,0 +1,184 @@
|
|||||||
|
zen_sync_config.json
|
||||||
|
|
||||||
|
# Byte-compiled / optimized / DLL files
|
||||||
|
__pycache__/
|
||||||
|
*.py[cod]
|
||||||
|
*$py.class
|
||||||
|
|
||||||
|
# C extensions
|
||||||
|
*.so
|
||||||
|
|
||||||
|
# Distribution / packaging
|
||||||
|
.Python
|
||||||
|
build/
|
||||||
|
develop-eggs/
|
||||||
|
dist/
|
||||||
|
downloads/
|
||||||
|
eggs/
|
||||||
|
.eggs/
|
||||||
|
lib/
|
||||||
|
lib64/
|
||||||
|
parts/
|
||||||
|
sdist/
|
||||||
|
var/
|
||||||
|
wheels/
|
||||||
|
share/python-wheels/
|
||||||
|
*.egg-info/
|
||||||
|
.installed.cfg
|
||||||
|
*.egg
|
||||||
|
MANIFEST
|
||||||
|
|
||||||
|
# PyInstaller
|
||||||
|
# Usually these files are written by a python script from a template
|
||||||
|
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
||||||
|
*.manifest
|
||||||
|
*.spec
|
||||||
|
|
||||||
|
# Installer logs
|
||||||
|
pip-log.txt
|
||||||
|
pip-delete-this-directory.txt
|
||||||
|
|
||||||
|
# Unit test / coverage reports
|
||||||
|
htmlcov/
|
||||||
|
.tox/
|
||||||
|
.nox/
|
||||||
|
.coverage
|
||||||
|
.coverage.*
|
||||||
|
.cache
|
||||||
|
nosetests.xml
|
||||||
|
coverage.xml
|
||||||
|
*.cover
|
||||||
|
*.py,cover
|
||||||
|
.hypothesis/
|
||||||
|
.pytest_cache/
|
||||||
|
cover/
|
||||||
|
|
||||||
|
# Translations
|
||||||
|
*.mo
|
||||||
|
*.pot
|
||||||
|
|
||||||
|
# Django stuff:
|
||||||
|
*.log
|
||||||
|
local_settings.py
|
||||||
|
db.sqlite3
|
||||||
|
db.sqlite3-journal
|
||||||
|
|
||||||
|
# Flask stuff:
|
||||||
|
instance/
|
||||||
|
.webassets-cache
|
||||||
|
|
||||||
|
# Scrapy stuff:
|
||||||
|
.scrapy
|
||||||
|
|
||||||
|
# Sphinx documentation
|
||||||
|
docs/_build/
|
||||||
|
|
||||||
|
# PyBuilder
|
||||||
|
.pybuilder/
|
||||||
|
target/
|
||||||
|
|
||||||
|
# Jupyter Notebook
|
||||||
|
.ipynb_checkpoints
|
||||||
|
|
||||||
|
# IPython
|
||||||
|
profile_default/
|
||||||
|
ipython_config.py
|
||||||
|
|
||||||
|
# pyenv
|
||||||
|
# For a library or package, you might want to ignore these files since the code is
|
||||||
|
# intended to run in multiple environments; otherwise, check them in:
|
||||||
|
# .python-version
|
||||||
|
|
||||||
|
# pipenv
|
||||||
|
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
|
||||||
|
# However, in case of collaboration, if having platform-specific dependencies or dependencies
|
||||||
|
# having no cross-platform support, pipenv may install dependencies that don't work, or not
|
||||||
|
# install all needed dependencies.
|
||||||
|
#Pipfile.lock
|
||||||
|
|
||||||
|
# UV
|
||||||
|
# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
|
||||||
|
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
||||||
|
# commonly ignored for libraries.
|
||||||
|
#uv.lock
|
||||||
|
|
||||||
|
# poetry
|
||||||
|
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
|
||||||
|
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
||||||
|
# commonly ignored for libraries.
|
||||||
|
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
|
||||||
|
#poetry.lock
|
||||||
|
|
||||||
|
# pdm
|
||||||
|
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
|
||||||
|
#pdm.lock
|
||||||
|
# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
|
||||||
|
# in version control.
|
||||||
|
# https://pdm.fming.dev/latest/usage/project/#working-with-version-control
|
||||||
|
.pdm.toml
|
||||||
|
.pdm-python
|
||||||
|
.pdm-build/
|
||||||
|
|
||||||
|
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
|
||||||
|
__pypackages__/
|
||||||
|
|
||||||
|
# Celery stuff
|
||||||
|
celerybeat-schedule
|
||||||
|
celerybeat.pid
|
||||||
|
|
||||||
|
# SageMath parsed files
|
||||||
|
*.sage.py
|
||||||
|
|
||||||
|
# Environments
|
||||||
|
.env
|
||||||
|
.venv
|
||||||
|
env/
|
||||||
|
venv/
|
||||||
|
ENV/
|
||||||
|
env.bak/
|
||||||
|
venv.bak/
|
||||||
|
|
||||||
|
# Spyder project settings
|
||||||
|
.spyderproject
|
||||||
|
.spyproject
|
||||||
|
|
||||||
|
# Rope project settings
|
||||||
|
.ropeproject
|
||||||
|
|
||||||
|
# mkdocs documentation
|
||||||
|
/site
|
||||||
|
|
||||||
|
# mypy
|
||||||
|
.mypy_cache/
|
||||||
|
.dmypy.json
|
||||||
|
dmypy.json
|
||||||
|
|
||||||
|
# Pyre type checker
|
||||||
|
.pyre/
|
||||||
|
|
||||||
|
# pytype static type analyzer
|
||||||
|
.pytype/
|
||||||
|
|
||||||
|
# Cython debug symbols
|
||||||
|
cython_debug/
|
||||||
|
|
||||||
|
# PyCharm
|
||||||
|
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
|
||||||
|
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
|
||||||
|
# and can be added to the global gitignore or merged into this file. For a more nuclear
|
||||||
|
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
|
||||||
|
#.idea/
|
||||||
|
|
||||||
|
# Visual Studio Code
|
||||||
|
# Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore
|
||||||
|
# that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
|
||||||
|
# and can be added to the global gitignore or merged into this file. However, if you prefer,
|
||||||
|
# you could uncomment the following to ignore the enitre vscode folder
|
||||||
|
# .vscode/
|
||||||
|
|
||||||
|
# Ruff stuff:
|
||||||
|
.ruff_cache/
|
||||||
|
|
||||||
|
# PyPI configuration file
|
||||||
|
.pypirc
|
||||||
|
|
||||||
21
LICENSE
Normal file
21
LICENSE
Normal file
@@ -0,0 +1,21 @@
|
|||||||
|
MIT License
|
||||||
|
|
||||||
|
Copyright (c) 2025 Yuzu
|
||||||
|
|
||||||
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||||
|
of this software and associated documentation files (the "Software"), to deal
|
||||||
|
in the Software without restriction, including without limitation the rights
|
||||||
|
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||||
|
copies of the Software, and to permit persons to whom the Software is
|
||||||
|
furnished to do so, subject to the following conditions:
|
||||||
|
|
||||||
|
The above copyright notice and this permission notice shall be included in all
|
||||||
|
copies or substantial portions of the Software.
|
||||||
|
|
||||||
|
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||||
|
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||||
|
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||||
|
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||||
|
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||||
|
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||||
|
SOFTWARE.
|
||||||
81
README.md
Normal file
81
README.md
Normal file
@@ -0,0 +1,81 @@
|
|||||||
|
# 🧘♀️ Zen-Sync
|
||||||
|
|
||||||
|
A Windows only command-line tool for syncing [Zen Browser](https://zen-browser.app/) data with S3-compatible storage services.
|
||||||
|
|
||||||
|
## 🤔 What it does
|
||||||
|
|
||||||
|
Since Zen Browser doesn't have proper profile sync yet, this is my quick solution built in a few hours to keep my stuffs in sync across multiple machines.
|
||||||
|
|
||||||
|
It backs up all the important stuff to any S3-compatible cloud storage so you can restore or "sync" your profile anywhere. No more manually dragging around profile folders every time you edit a settings 🥹🥹😭. I'm so done with that.
|
||||||
|
|
||||||
|
The default (customizable) setting skips session cookies, temporary storage, and other data because sites I visit can detect copied sessions through fingerprinting and will invalidate them.
|
||||||
|
|
||||||
|
## ✨ Features
|
||||||
|
|
||||||
|
- 🔄 **Bidirectional sync** between local and S3 storage
|
||||||
|
- 🔍 **Filtering** - only syncs important files, excludes cache and temporary data
|
||||||
|
- ⚡ **"Incremental" sync** - only uploads/downloads changed files
|
||||||
|
- 🔗 **Custom S3 endpoints** - works with any S3-compatible service
|
||||||
|
|
||||||
|
## 📋 What gets synced by default
|
||||||
|
|
||||||
|
**Included:**
|
||||||
|
- 📁 Profile configuration (`profiles.ini`, `installs.ini`, `compatibility.ini`)
|
||||||
|
- 🗃️ Profile Groups databases (`Profile Groups/*.sqlite`)
|
||||||
|
- 📚 Bookmarks (`places.sqlite`, `bookmarks.html`)
|
||||||
|
- 🔒 Saved passwords and certificates (`key4.db`, `cert9.db`, `logins.json`)
|
||||||
|
- 🧩 Extensions and their settings (`extensions.json`, `extension-*.json`)
|
||||||
|
- 🎨 Custom themes and CSS (`zen-*.json`, `zen-*.css`, `userChrome.css`, `userContent.css`)
|
||||||
|
- ⚙️ Browser preferences (`prefs.js`, `user.js`)
|
||||||
|
- 🔍 Search engine settings (`search.json.mozlz4`)
|
||||||
|
- 🖼️ Favicons (`favicons.sqlite`)
|
||||||
|
- 📂 Chrome folder customizations (`chrome/**/*`)
|
||||||
|
- 📔 and other files from customizable ruleset
|
||||||
|
|
||||||
|
**Excluded:**
|
||||||
|
- 🗑️ Cache files (`cache2/*`, `thumbnails/*`, `shader-cache/*`)
|
||||||
|
- 📜 Logs and crash reports (`logs/*`, `crashes/*`, `minidumps/*`)
|
||||||
|
- 🔒 Lock files (`*.lock`, `*.lck`, `parent.lock`)
|
||||||
|
- 💾 Temporary storage (`storage/temporary/*`, `storage/*/ls/*`)
|
||||||
|
- 📋 Session data (`sessionstore.jsonlz4`, `sessionCheckpoints.json`)
|
||||||
|
- 🍪 Session cookies (`cookies.sqlite*`)
|
||||||
|
- 🛡️ Temporary browsing data (`webappsstore.sqlite*`, `safebrowsing/*`)
|
||||||
|
|
||||||
|
Use `--help` with any command for detailed options.
|
||||||
|
|
||||||
|
## 🚀 Quick Start
|
||||||
|
|
||||||
|
1. ⚙️ **Configure your S3 settings:**
|
||||||
|
```bash
|
||||||
|
python zensync.py configure --bucket your-bucket-name --endpoint-url https://your-s3-endpoint.com
|
||||||
|
```
|
||||||
|
|
||||||
|
or just run ```python zensync.py configure``` then edit the configuration json manually.
|
||||||
|
|
||||||
|
2. ⬆️ **Upload your profiles:**
|
||||||
|
```bash
|
||||||
|
python zensync.py upload
|
||||||
|
```
|
||||||
|
|
||||||
|
3. ⬇️ **Download profiles on another machine:**
|
||||||
|
```bash
|
||||||
|
python zensync.py download
|
||||||
|
```
|
||||||
|
|
||||||
|
4. 🔄 **Two-way sync:**
|
||||||
|
```bash
|
||||||
|
python zensync.py sync
|
||||||
|
```
|
||||||
|
|
||||||
|
## Main Commands 🎮
|
||||||
|
|
||||||
|
- ⚙️ `configure` - Set up S3 credentials and paths
|
||||||
|
- ⬆️ `upload` - Backup profiles to S3
|
||||||
|
- ⬇️ `download` - Restore profiles from S3
|
||||||
|
- 🔄 `sync` - Bidirectional synchronization
|
||||||
|
- 📋 `list-profiles` - Show available local profiles
|
||||||
|
- ℹ️ `profile-info` - Display profile system information
|
||||||
|
|
||||||
|
## 📝 Configuration
|
||||||
|
|
||||||
|
Settings are stored in `zen_sync_config.json`.
|
||||||
259
cli.py
Normal file
259
cli.py
Normal file
@@ -0,0 +1,259 @@
|
|||||||
|
import argparse
|
||||||
|
import sys
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
from config import ZenSyncConfig
|
||||||
|
from sync import ZenS3Sync
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
def create_parser():
|
||||||
|
parser = argparse.ArgumentParser(
|
||||||
|
description="Zen Browser Profile S3 Sync Tool",
|
||||||
|
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||||
|
epilog="""
|
||||||
|
Examples:
|
||||||
|
zensync upload --bucket my-backup-bucket
|
||||||
|
zensync download --bucket my-backup-bucket
|
||||||
|
zensync sync --bucket my-backup-bucket
|
||||||
|
zensync configure --bucket my-bucket --endpoint-url http://localhost:9000
|
||||||
|
zensync list-profiles
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
|
||||||
|
parser.add_argument('--config', default='zen_sync_config.json', help='Configuration file path')
|
||||||
|
parser.add_argument('--roaming-path', help='Override Zen roaming data path')
|
||||||
|
parser.add_argument('--local-path', help='Override Zen local data path')
|
||||||
|
parser.add_argument('--verbose', '-v', action='store_true', help='Enable verbose logging')
|
||||||
|
|
||||||
|
subparsers = parser.add_subparsers(dest='command', help='Available commands')
|
||||||
|
|
||||||
|
# Upload command
|
||||||
|
upload_parser = subparsers.add_parser('upload', help='Upload profiles to S3')
|
||||||
|
upload_parser.add_argument('--bucket', help='S3 bucket name')
|
||||||
|
upload_parser.add_argument('--prefix', default='zen-profiles/', help='S3 key prefix')
|
||||||
|
upload_parser.add_argument('--dry-run', action='store_true', help='Show what would be uploaded')
|
||||||
|
upload_parser.add_argument('--no-cache', action='store_true', help='Disable cache data upload')
|
||||||
|
upload_parser.add_argument('--force-full', action='store_true', help='Force full upload')
|
||||||
|
upload_parser.add_argument('--cleanup', action='store_true', help='Remove S3 files that no longer exist locally')
|
||||||
|
|
||||||
|
# Download command
|
||||||
|
download_parser = subparsers.add_parser('download', help='Download profiles from S3')
|
||||||
|
download_parser.add_argument('--bucket', help='S3 bucket name')
|
||||||
|
download_parser.add_argument('--prefix', default='zen-profiles/', help='S3 key prefix')
|
||||||
|
download_parser.add_argument('--dry-run', action='store_true', help='Show what would be downloaded')
|
||||||
|
download_parser.add_argument('--no-cache', action='store_true', help='Disable cache data download')
|
||||||
|
download_parser.add_argument('--force-full', action='store_true', help='Force full download')
|
||||||
|
download_parser.add_argument('--cleanup', action='store_true', help='Remove local files that no longer exist in S3')
|
||||||
|
|
||||||
|
# Sync command
|
||||||
|
sync_parser = subparsers.add_parser('sync', help='Bidirectional sync between local and S3')
|
||||||
|
sync_parser.add_argument('--bucket', help='S3 bucket name')
|
||||||
|
sync_parser.add_argument('--prefix', default='zen-profiles/', help='S3 key prefix')
|
||||||
|
sync_parser.add_argument('--dry-run', action='store_true', help='Show what would be synced')
|
||||||
|
sync_parser.add_argument('--no-cache', action='store_true', help='Disable cache data sync')
|
||||||
|
sync_parser.add_argument('--cleanup', action='store_true', help='Remove orphaned files')
|
||||||
|
|
||||||
|
# List profiles command
|
||||||
|
subparsers.add_parser('list-profiles', help='List available local profiles')
|
||||||
|
|
||||||
|
# Profile info command
|
||||||
|
subparsers.add_parser('profile-info', help='Show profile system information')
|
||||||
|
|
||||||
|
# Configure command
|
||||||
|
config_parser = subparsers.add_parser('configure', help='Configure sync settings')
|
||||||
|
config_parser.add_argument('--bucket', help='Set S3 bucket name')
|
||||||
|
config_parser.add_argument('--region', help='Set AWS region')
|
||||||
|
config_parser.add_argument('--endpoint-url', help='Set S3-compatible service endpoint')
|
||||||
|
config_parser.add_argument('--access-key', help='Set AWS access key ID')
|
||||||
|
config_parser.add_argument('--secret-key', help='Set AWS secret access key')
|
||||||
|
config_parser.add_argument('--profile', help='Set AWS profile name')
|
||||||
|
config_parser.add_argument('--roaming-path', help='Set Zen roaming data path')
|
||||||
|
config_parser.add_argument('--local-path', help='Set Zen local data path')
|
||||||
|
config_parser.add_argument('--auto-detect', action='store_true', help='Auto-detect Zen browser paths')
|
||||||
|
config_parser.add_argument('--enable-cache-sync', action='store_true', help='Enable cache data sync')
|
||||||
|
config_parser.add_argument('--disable-cache-sync', action='store_true', help='Disable cache data sync')
|
||||||
|
config_parser.add_argument('--disable-metadata', action='store_true', help='Disable S3 metadata')
|
||||||
|
config_parser.add_argument('--enable-metadata', action='store_true', help='Enable S3 metadata')
|
||||||
|
config_parser.add_argument('--signature-version', choices=['s3', 's3v4'], help='Set AWS signature version')
|
||||||
|
|
||||||
|
return parser
|
||||||
|
|
||||||
|
def handle_configure(args, config):
|
||||||
|
"""Handle configure command"""
|
||||||
|
if args.bucket:
|
||||||
|
config.config['aws']['bucket'] = args.bucket
|
||||||
|
if args.region:
|
||||||
|
config.config['aws']['region'] = args.region
|
||||||
|
if getattr(args, 'endpoint_url', None):
|
||||||
|
config.config['aws']['endpoint_url'] = args.endpoint_url
|
||||||
|
logger.info(f"Using custom S3 endpoint: {args.endpoint_url}")
|
||||||
|
if args.access_key:
|
||||||
|
config.config['aws']['access_key_id'] = args.access_key
|
||||||
|
logger.warning("Storing AWS access key in config file")
|
||||||
|
if args.secret_key:
|
||||||
|
config.config['aws']['secret_access_key'] = args.secret_key
|
||||||
|
logger.warning("Storing AWS secret key in config file")
|
||||||
|
if args.profile:
|
||||||
|
config.config['aws']['profile'] = args.profile
|
||||||
|
config.config['aws']['access_key_id'] = ""
|
||||||
|
config.config['aws']['secret_access_key'] = ""
|
||||||
|
logger.info(f"Configured to use AWS profile: {args.profile}")
|
||||||
|
if args.roaming_path:
|
||||||
|
config.config['sync']['zen_roaming_path'] = args.roaming_path
|
||||||
|
if args.local_path:
|
||||||
|
config.config['sync']['zen_local_path'] = args.local_path
|
||||||
|
|
||||||
|
if args.auto_detect:
|
||||||
|
auto_paths = config.auto_detect_zen_paths()
|
||||||
|
if auto_paths['roaming']:
|
||||||
|
config.config['sync']['zen_roaming_path'] = auto_paths['roaming']
|
||||||
|
print(f"Auto-detected roaming path: {auto_paths['roaming']}")
|
||||||
|
if auto_paths['local']:
|
||||||
|
config.config['sync']['zen_local_path'] = auto_paths['local']
|
||||||
|
print(f"Auto-detected local path: {auto_paths['local']}")
|
||||||
|
|
||||||
|
if args.enable_cache_sync:
|
||||||
|
config.config['sync']['sync_cache_data'] = True
|
||||||
|
if args.disable_cache_sync:
|
||||||
|
config.config['sync']['sync_cache_data'] = False
|
||||||
|
if getattr(args, 'disable_metadata', False):
|
||||||
|
config.config['aws']['disable_metadata'] = True
|
||||||
|
logger.info("S3 metadata disabled")
|
||||||
|
if getattr(args, 'enable_metadata', False):
|
||||||
|
config.config['aws']['disable_metadata'] = False
|
||||||
|
logger.info("S3 metadata enabled")
|
||||||
|
if getattr(args, 'signature_version', None):
|
||||||
|
config.config['aws']['signature_version'] = args.signature_version
|
||||||
|
logger.info(f"AWS signature version set to: {args.signature_version}")
|
||||||
|
|
||||||
|
config.save_config()
|
||||||
|
|
||||||
|
display_config = json.loads(json.dumps(config.config))
|
||||||
|
if display_config['aws'].get('secret_access_key'):
|
||||||
|
display_config['aws']['secret_access_key'] = "***HIDDEN***"
|
||||||
|
|
||||||
|
print("\nConfiguration updated:")
|
||||||
|
print(json.dumps(display_config, indent=2))
|
||||||
|
|
||||||
|
def handle_list_profiles(sync):
|
||||||
|
"""Handle list-profiles command"""
|
||||||
|
profiles = sync.list_profiles()
|
||||||
|
if profiles:
|
||||||
|
print(f"\nAvailable Zen Browser Profiles:")
|
||||||
|
print("=" * 70)
|
||||||
|
for profile_id, info in profiles.items():
|
||||||
|
status = " (Default)" if info['is_default'] else ""
|
||||||
|
print(f"• {info['name']}{status}")
|
||||||
|
print(f" Profile ID: {profile_id}")
|
||||||
|
print(f" Path: {info['path']}")
|
||||||
|
print(f" Store ID: {info.get('store_id', 'N/A')}")
|
||||||
|
print(f" Full Path: {info['full_path']}")
|
||||||
|
print()
|
||||||
|
else:
|
||||||
|
print("No profiles found")
|
||||||
|
|
||||||
|
def handle_profile_info(sync):
|
||||||
|
"""Handle profile-info command"""
|
||||||
|
info = sync.get_profile_info()
|
||||||
|
print(f"\nZen Browser Profile System Information:")
|
||||||
|
print("=" * 70)
|
||||||
|
print(f"System Type: {info['system_type']}")
|
||||||
|
print("\nPaths:")
|
||||||
|
for path_name, path_value in info['paths'].items():
|
||||||
|
print(f" {path_name}: {path_value}")
|
||||||
|
|
||||||
|
print(f"\nProfiles Found: {len(info['profiles'])}")
|
||||||
|
if info['profiles']:
|
||||||
|
for profile_id, profile_info in info['profiles'].items():
|
||||||
|
status = " (Default)" if profile_info['is_default'] else ""
|
||||||
|
print(f" • {profile_info['name']}{status}")
|
||||||
|
|
||||||
|
if 'profile_groups' in info:
|
||||||
|
print(f"\nProfile Groups:")
|
||||||
|
if info['profile_groups'].get('exists'):
|
||||||
|
print(f" Path: {info['profile_groups']['path']}")
|
||||||
|
print(f" Databases: {', '.join(info['profile_groups'].get('databases', []))}")
|
||||||
|
else:
|
||||||
|
print(" Not found")
|
||||||
|
|
||||||
|
def run_cli():
|
||||||
|
"""Main CLI entry point"""
|
||||||
|
parser = create_parser()
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
if args.verbose:
|
||||||
|
logging.getLogger().setLevel(logging.DEBUG)
|
||||||
|
|
||||||
|
config = ZenSyncConfig(args.config)
|
||||||
|
|
||||||
|
if args.roaming_path:
|
||||||
|
config.config['sync']['zen_roaming_path'] = args.roaming_path
|
||||||
|
if args.local_path:
|
||||||
|
config.config['sync']['zen_local_path'] = args.local_path
|
||||||
|
|
||||||
|
if args.command == 'configure':
|
||||||
|
handle_configure(args, config)
|
||||||
|
return
|
||||||
|
|
||||||
|
if args.command in ['upload', 'download', 'sync']:
|
||||||
|
if args.bucket:
|
||||||
|
config.config['aws']['bucket'] = args.bucket
|
||||||
|
if args.prefix:
|
||||||
|
config.config['aws']['prefix'] = args.prefix
|
||||||
|
if hasattr(args, 'no_cache') and args.no_cache:
|
||||||
|
config.config['sync']['sync_cache_data'] = False
|
||||||
|
logger.info("Cache sync disabled for this operation")
|
||||||
|
|
||||||
|
if not args.command:
|
||||||
|
parser.print_help()
|
||||||
|
return
|
||||||
|
|
||||||
|
try:
|
||||||
|
require_s3 = args.command not in ['list-profiles', 'profile-info']
|
||||||
|
if args.command in ['upload', 'download', 'sync'] and hasattr(args, 'dry_run') and args.dry_run:
|
||||||
|
require_s3 = True
|
||||||
|
logger.info("Dry run mode: Will analyze existing S3 objects")
|
||||||
|
|
||||||
|
sync = ZenS3Sync(config, require_s3=require_s3)
|
||||||
|
|
||||||
|
if args.command == 'upload':
|
||||||
|
incremental = not getattr(args, 'force_full', False)
|
||||||
|
cleanup = getattr(args, 'cleanup', False)
|
||||||
|
success = sync.upload_to_s3(
|
||||||
|
dry_run=args.dry_run,
|
||||||
|
incremental=incremental,
|
||||||
|
cleanup=cleanup
|
||||||
|
)
|
||||||
|
sys.exit(0 if success else 1)
|
||||||
|
|
||||||
|
elif args.command == 'download':
|
||||||
|
incremental = not getattr(args, 'force_full', False)
|
||||||
|
cleanup = getattr(args, 'cleanup', False)
|
||||||
|
success = sync.download_from_s3(
|
||||||
|
dry_run=args.dry_run,
|
||||||
|
incremental=incremental,
|
||||||
|
cleanup=cleanup
|
||||||
|
)
|
||||||
|
sys.exit(0 if success else 1)
|
||||||
|
|
||||||
|
elif args.command == 'sync':
|
||||||
|
cleanup = getattr(args, 'cleanup', False)
|
||||||
|
success = sync.sync_bidirectional(
|
||||||
|
dry_run=args.dry_run,
|
||||||
|
cleanup=cleanup
|
||||||
|
)
|
||||||
|
sys.exit(0 if success else 1)
|
||||||
|
|
||||||
|
elif args.command == 'list-profiles':
|
||||||
|
handle_list_profiles(sync)
|
||||||
|
|
||||||
|
elif args.command == 'profile-info':
|
||||||
|
handle_profile_info(sync)
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Error: {e}")
|
||||||
|
if args.verbose:
|
||||||
|
import traceback
|
||||||
|
traceback.print_exc()
|
||||||
|
sys.exit(1)
|
||||||
108
config.py
Normal file
108
config.py
Normal file
@@ -0,0 +1,108 @@
|
|||||||
|
import os
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import platform
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Dict
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
class ZenSyncConfig:
|
||||||
|
"""Configuration management for Zen sync operations"""
|
||||||
|
|
||||||
|
def __init__(self, config_file: str = "zen_sync_config.json"):
|
||||||
|
self.config_file = config_file
|
||||||
|
self.config = self.load_config()
|
||||||
|
|
||||||
|
def load_config(self) -> Dict:
|
||||||
|
"""Load configuration from file or create default"""
|
||||||
|
default_config = {
|
||||||
|
"aws": {
|
||||||
|
"region": "us-east-1",
|
||||||
|
"bucket": "",
|
||||||
|
"prefix": "zen-profiles/",
|
||||||
|
"endpoint_url": "",
|
||||||
|
"disable_metadata": False,
|
||||||
|
"signature_version": "s3v4",
|
||||||
|
"access_key_id": "",
|
||||||
|
"secret_access_key": "",
|
||||||
|
"profile": ""
|
||||||
|
},
|
||||||
|
"sync": {
|
||||||
|
"zen_roaming_path": "",
|
||||||
|
"zen_local_path": "",
|
||||||
|
"sync_cache_data": False,
|
||||||
|
"exclude_patterns": [
|
||||||
|
"*.lock", "*.lck", "*-wal", "*-shm", "*-journal",
|
||||||
|
"parent.lock", "cookies.sqlite*", "webappsstore.sqlite*",
|
||||||
|
"storage/temporary/*", "storage/default/*/ls/*", "storage/permanent/*/ls/*",
|
||||||
|
"cache2/*", "jumpListCache/*", "offlineCache/*", "thumbnails/*",
|
||||||
|
"crashes/*", "minidumps/*", "shader-cache/*", "startupCache/*",
|
||||||
|
"safebrowsing/*", "logs/*", "sessionstore-backups/previous.jsonlz4",
|
||||||
|
"sessionstore-backups/upgrade.jsonlz4-*",
|
||||||
|
"Profile Groups/*.sqlite-shm", "Profile Groups/*.sqlite-wal"
|
||||||
|
],
|
||||||
|
"include_important": [
|
||||||
|
"*.ini", "prefs.js", "user.js", "userChrome.css", "userContent.css",
|
||||||
|
"bookmarks.html", "places.sqlite", "favicons.sqlite", "key4.db",
|
||||||
|
"cert9.db", "extensions.json", "extension-settings.json",
|
||||||
|
"extension-preferences.json", "search.json.mozlz4", "handlers.json",
|
||||||
|
"containers.json", "zen-*.json", "zen-*.css", "chrome/**/*",
|
||||||
|
"profiles.ini", "installs.ini", "Profile Groups/*.sqlite",
|
||||||
|
"zen-keyboard-shortcuts.json", "zen-themes.json", "sessionstore.jsonlz4",
|
||||||
|
"sessionCheckpoints.json", "logins.json", "compatibility.ini"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if os.path.exists(self.config_file):
|
||||||
|
try:
|
||||||
|
with open(self.config_file, 'r') as f:
|
||||||
|
config = json.load(f)
|
||||||
|
|
||||||
|
# Merge with defaults for missing keys
|
||||||
|
for key in default_config:
|
||||||
|
if key not in config:
|
||||||
|
config[key] = default_config[key]
|
||||||
|
elif isinstance(default_config[key], dict):
|
||||||
|
for subkey in default_config[key]:
|
||||||
|
if subkey not in config[key]:
|
||||||
|
config[key][subkey] = default_config[key][subkey]
|
||||||
|
return config
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(f"Error loading config file: {e}. Using defaults.")
|
||||||
|
|
||||||
|
return default_config
|
||||||
|
|
||||||
|
def auto_detect_zen_paths(self) -> Dict[str, str]:
|
||||||
|
"""Auto-detect Zen browser installation paths"""
|
||||||
|
system = platform.system()
|
||||||
|
paths = {"roaming": "", "local": ""}
|
||||||
|
|
||||||
|
if system == "Windows":
|
||||||
|
roaming = os.path.expandvars(r"%APPDATA%\zen")
|
||||||
|
local = os.path.expandvars(r"%LOCALAPPDATA%\zen")
|
||||||
|
elif system == "Darwin":
|
||||||
|
home = os.path.expanduser("~")
|
||||||
|
roaming = os.path.join(home, "Library", "Application Support", "zen")
|
||||||
|
local = os.path.join(home, "Library", "Caches", "zen")
|
||||||
|
else:
|
||||||
|
home = os.path.expanduser("~")
|
||||||
|
roaming = os.path.join(home, ".zen")
|
||||||
|
local = os.path.join(home, ".cache", "zen")
|
||||||
|
|
||||||
|
if os.path.exists(roaming):
|
||||||
|
paths["roaming"] = roaming
|
||||||
|
if os.path.exists(local):
|
||||||
|
paths["local"] = local
|
||||||
|
|
||||||
|
return paths
|
||||||
|
|
||||||
|
def save_config(self):
|
||||||
|
"""Save current configuration to file"""
|
||||||
|
try:
|
||||||
|
with open(self.config_file, 'w') as f:
|
||||||
|
json.dump(self.config, f, indent=2)
|
||||||
|
logger.info(f"Configuration saved to {self.config_file}")
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Error saving config: {e}")
|
||||||
666
sync.py
Normal file
666
sync.py
Normal file
@@ -0,0 +1,666 @@
|
|||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import logging
|
||||||
|
import configparser
|
||||||
|
from pathlib import Path
|
||||||
|
from datetime import datetime
|
||||||
|
from typing import List, Dict, Set, Tuple, Optional
|
||||||
|
import boto3
|
||||||
|
from botocore.exceptions import ClientError, NoCredentialsError
|
||||||
|
from tqdm import tqdm
|
||||||
|
import fnmatch
|
||||||
|
from boto3.session import Config
|
||||||
|
|
||||||
|
from config import ZenSyncConfig
|
||||||
|
from utils import calculate_file_hash, format_size
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
class ZenS3Sync:
|
||||||
|
"""Main sync class for Zen Browser profiles"""
|
||||||
|
|
||||||
|
def __init__(self, config: ZenSyncConfig, require_s3: bool = True):
|
||||||
|
self.config = config
|
||||||
|
self.s3_client = None
|
||||||
|
self.bucket = config.config['aws']['bucket']
|
||||||
|
self.prefix = config.config['aws']['prefix']
|
||||||
|
|
||||||
|
self._initialize_paths()
|
||||||
|
|
||||||
|
self.exclude_patterns = config.config['sync']['exclude_patterns']
|
||||||
|
self.include_patterns = config.config['sync']['include_important']
|
||||||
|
|
||||||
|
if require_s3:
|
||||||
|
if not self.bucket:
|
||||||
|
raise ValueError("S3 bucket name must be configured")
|
||||||
|
self._init_s3_client()
|
||||||
|
|
||||||
|
def _initialize_paths(self):
|
||||||
|
"""Initialize Zen browser paths"""
|
||||||
|
sync_config = self.config.config['sync']
|
||||||
|
auto_paths = self.config.auto_detect_zen_paths()
|
||||||
|
|
||||||
|
self.zen_roaming_path = Path(sync_config['zen_roaming_path'] or auto_paths['roaming'] or '')
|
||||||
|
self.zen_local_path = Path(sync_config['zen_local_path'] or auto_paths['local'] or '')
|
||||||
|
|
||||||
|
logger.info(f"Zen Browser paths:")
|
||||||
|
logger.info(f" Roaming: {self.zen_roaming_path}")
|
||||||
|
logger.info(f" Local: {self.zen_local_path}")
|
||||||
|
|
||||||
|
if not self.zen_roaming_path.exists():
|
||||||
|
logger.warning(f"Roaming path does not exist: {self.zen_roaming_path}")
|
||||||
|
if not self.zen_local_path.exists():
|
||||||
|
logger.warning(f"Local path does not exist: {self.zen_local_path}")
|
||||||
|
|
||||||
|
def _init_s3_client(self):
|
||||||
|
"""Initialize S3 client"""
|
||||||
|
try:
|
||||||
|
aws_config = self.config.config['aws']
|
||||||
|
|
||||||
|
session_kwargs = {}
|
||||||
|
client_kwargs = {'region_name': aws_config['region']}
|
||||||
|
|
||||||
|
config_settings = {}
|
||||||
|
if aws_config.get('signature_version'):
|
||||||
|
config_settings['signature_version'] = aws_config['signature_version']
|
||||||
|
|
||||||
|
if aws_config.get('endpoint_url'):
|
||||||
|
client_kwargs['endpoint_url'] = aws_config['endpoint_url']
|
||||||
|
config_settings['s3'] = {'addressing_style': 'path'}
|
||||||
|
logger.info(f"Using S3 endpoint: {aws_config['endpoint_url']}")
|
||||||
|
|
||||||
|
if config_settings:
|
||||||
|
client_kwargs['config'] = Config(**config_settings)
|
||||||
|
|
||||||
|
if aws_config.get('profile'):
|
||||||
|
session_kwargs['profile_name'] = aws_config['profile']
|
||||||
|
logger.info(f"Using AWS profile: {aws_config['profile']}")
|
||||||
|
elif aws_config.get('access_key_id') and aws_config.get('secret_access_key'):
|
||||||
|
client_kwargs.update({
|
||||||
|
'aws_access_key_id': aws_config['access_key_id'],
|
||||||
|
'aws_secret_access_key': aws_config['secret_access_key']
|
||||||
|
})
|
||||||
|
logger.warning("Using credentials from config file")
|
||||||
|
|
||||||
|
if session_kwargs:
|
||||||
|
session = boto3.Session(**session_kwargs)
|
||||||
|
self.s3_client = session.client('s3', **client_kwargs)
|
||||||
|
else:
|
||||||
|
self.s3_client = boto3.client('s3', **client_kwargs)
|
||||||
|
|
||||||
|
self.s3_client.head_bucket(Bucket=self.bucket)
|
||||||
|
logger.info(f"Connected to S3, bucket: {self.bucket}")
|
||||||
|
|
||||||
|
except NoCredentialsError:
|
||||||
|
logger.error("AWS credentials not found")
|
||||||
|
sys.exit(1)
|
||||||
|
except ClientError as e:
|
||||||
|
if e.response['Error']['Code'] == '404':
|
||||||
|
logger.error(f"S3 bucket '{self.bucket}' not found")
|
||||||
|
else:
|
||||||
|
logger.error(f"Error connecting to S3: {e}")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
def _get_s3_key(self, file_path: Path, base_path: Path, path_type: str) -> str:
|
||||||
|
relative_path = file_path.relative_to(base_path)
|
||||||
|
if path_type in ['roaming', 'local']:
|
||||||
|
return f"{self.prefix}{path_type}/{relative_path}".replace('\\', '/')
|
||||||
|
return f"{self.prefix}{relative_path}".replace('\\', '/')
|
||||||
|
|
||||||
|
def _get_relative_s3_key(self, file_path: Path, base_path: Path, path_type: str) -> str:
|
||||||
|
relative_path = file_path.relative_to(base_path)
|
||||||
|
if path_type in ['roaming', 'local']:
|
||||||
|
return f"{path_type}/{relative_path}".replace('\\', '/')
|
||||||
|
return str(relative_path).replace('\\', '/')
|
||||||
|
|
||||||
|
def _get_download_path(self, relative_path: str) -> Optional[Path]:
|
||||||
|
if relative_path.startswith('roaming/'):
|
||||||
|
return self.zen_roaming_path / relative_path[8:] if self.zen_roaming_path else None
|
||||||
|
elif relative_path.startswith('local/'):
|
||||||
|
if self.zen_local_path and self.config.config['sync']['sync_cache_data']:
|
||||||
|
return self.zen_local_path / relative_path[6:]
|
||||||
|
return None
|
||||||
|
return self.zen_roaming_path / relative_path if self.zen_roaming_path else None
|
||||||
|
|
||||||
|
def _get_file_info(self, file_path: Path) -> Dict:
|
||||||
|
"""Get file information for comparison"""
|
||||||
|
try:
|
||||||
|
stat = file_path.stat()
|
||||||
|
return {
|
||||||
|
'size': stat.st_size,
|
||||||
|
'mtime': int(stat.st_mtime),
|
||||||
|
'hash': calculate_file_hash(file_path),
|
||||||
|
'exists': True
|
||||||
|
}
|
||||||
|
except (OSError, FileNotFoundError):
|
||||||
|
return {'exists': False}
|
||||||
|
|
||||||
|
def _files_are_different(self, local_info: Dict, s3_info: Dict) -> bool:
|
||||||
|
"""Compare local file with S3 object"""
|
||||||
|
if not local_info['exists'] or not s3_info['exists']:
|
||||||
|
return True
|
||||||
|
|
||||||
|
# Use hash comparison if available (apparently some s3 don't support putting custom metadata)
|
||||||
|
if (local_info.get('hash') and s3_info.get('hash') and
|
||||||
|
local_info['hash'] and s3_info['hash']):
|
||||||
|
are_different = local_info['hash'] != s3_info['hash']
|
||||||
|
if are_different:
|
||||||
|
logger.debug(f"Hash comparison: files different")
|
||||||
|
else:
|
||||||
|
logger.debug(f"Hash comparison: files identical")
|
||||||
|
return are_different
|
||||||
|
|
||||||
|
# Fallback to size comparison
|
||||||
|
if local_info['size'] != s3_info['size']:
|
||||||
|
logger.debug(f"Size comparison: files different")
|
||||||
|
return True
|
||||||
|
|
||||||
|
logger.debug(f"Size comparison: files identical")
|
||||||
|
return False
|
||||||
|
|
||||||
|
def _list_s3_objects(self) -> Dict[str, Dict]:
|
||||||
|
"""List all S3 objects with metadata"""
|
||||||
|
objects = {}
|
||||||
|
try:
|
||||||
|
paginator = self.s3_client.get_paginator('list_objects_v2')
|
||||||
|
pages = paginator.paginate(Bucket=self.bucket, Prefix=self.prefix)
|
||||||
|
|
||||||
|
for page in pages:
|
||||||
|
if 'Contents' in page:
|
||||||
|
for obj in page['Contents']:
|
||||||
|
relative_key = obj['Key'][len(self.prefix):]
|
||||||
|
|
||||||
|
obj_info = {
|
||||||
|
'size': obj['Size'],
|
||||||
|
'mtime': int(obj['LastModified'].timestamp()),
|
||||||
|
'etag': obj['ETag'].strip('"'),
|
||||||
|
'exists': True,
|
||||||
|
's3_key': obj['Key'],
|
||||||
|
'hash': None
|
||||||
|
}
|
||||||
|
|
||||||
|
# Try to get hash from metadata
|
||||||
|
try:
|
||||||
|
head_response = self.s3_client.head_object(Bucket=self.bucket, Key=obj['Key'])
|
||||||
|
if 'Metadata' in head_response and not self.config.config['aws'].get('disable_metadata', False):
|
||||||
|
metadata = head_response['Metadata']
|
||||||
|
if 'file-hash' in metadata:
|
||||||
|
obj_info['hash'] = metadata['file-hash']
|
||||||
|
elif 'file_hash' in metadata:
|
||||||
|
obj_info['hash'] = metadata['file_hash']
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
objects[relative_key] = obj_info
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Error listing S3 objects: {e}")
|
||||||
|
|
||||||
|
return objects
|
||||||
|
|
||||||
|
def _log_sync_analysis(self, upload_files: List, download_files: List, skip_files: List, delete_files: List = None):
|
||||||
|
total_upload_size = sum(item[2] for item in upload_files)
|
||||||
|
total_download_size = sum(item[2] for item in download_files)
|
||||||
|
total_skip_size = sum(item[2] for item in skip_files)
|
||||||
|
|
||||||
|
logger.info(f"Sync analysis:")
|
||||||
|
logger.info(f" Upload: {len(upload_files)} files ({format_size(total_upload_size)})")
|
||||||
|
logger.info(f" Download: {len(download_files)} files ({format_size(total_download_size)})")
|
||||||
|
logger.info(f" Skip: {len(skip_files)} files ({format_size(total_skip_size)})")
|
||||||
|
|
||||||
|
if delete_files:
|
||||||
|
total_delete_size = sum(item[2] for item in delete_files)
|
||||||
|
logger.info(f" Delete: {len(delete_files)} files ({format_size(total_delete_size)})")
|
||||||
|
|
||||||
|
def _process_files(self, files: List, action: str, dry_run: bool, processor_func) -> bool:
|
||||||
|
if not files:
|
||||||
|
return True
|
||||||
|
|
||||||
|
logger.info(f"{'[DRY RUN] ' if dry_run else ''}{action.capitalize()} {len(files)} files...")
|
||||||
|
success_count = 0
|
||||||
|
error_count = 0
|
||||||
|
|
||||||
|
with tqdm(total=len(files), desc=action.capitalize(), unit="file") as pbar:
|
||||||
|
for file_args in files:
|
||||||
|
try:
|
||||||
|
if not dry_run:
|
||||||
|
processor_func(*file_args)
|
||||||
|
success_count += 1
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Error {action} {file_args[0]}: {e}")
|
||||||
|
error_count += 1
|
||||||
|
pbar.update(1)
|
||||||
|
|
||||||
|
return error_count == 0
|
||||||
|
|
||||||
|
def should_include_file(self, file_path: Path, base_path: Path) -> bool:
|
||||||
|
"""Check if file should be included in sync"""
|
||||||
|
relative_path = file_path.relative_to(base_path)
|
||||||
|
str_path = str(relative_path).replace('\\', '/')
|
||||||
|
|
||||||
|
for pattern in self.exclude_patterns:
|
||||||
|
if fnmatch.fnmatch(str_path, pattern) or fnmatch.fnmatch(file_path.name, pattern):
|
||||||
|
return False
|
||||||
|
|
||||||
|
for pattern in self.include_patterns:
|
||||||
|
if fnmatch.fnmatch(str_path, pattern) or fnmatch.fnmatch(file_path.name, pattern):
|
||||||
|
return True
|
||||||
|
|
||||||
|
return True
|
||||||
|
|
||||||
|
def get_local_files(self) -> List[tuple]:
|
||||||
|
"""Get list of local files to sync"""
|
||||||
|
files = []
|
||||||
|
|
||||||
|
if self.zen_roaming_path and self.zen_roaming_path.exists():
|
||||||
|
roaming_files = self._scan_directory(self.zen_roaming_path, 'roaming')
|
||||||
|
files.extend(roaming_files)
|
||||||
|
logger.info(f"Found {len(roaming_files)} files in roaming directory")
|
||||||
|
else:
|
||||||
|
logger.error("Roaming directory not found")
|
||||||
|
return []
|
||||||
|
|
||||||
|
if (self.zen_local_path and self.zen_local_path.exists() and
|
||||||
|
self.config.config['sync']['sync_cache_data']):
|
||||||
|
local_files = self._scan_directory(self.zen_local_path, 'local')
|
||||||
|
files.extend(local_files)
|
||||||
|
logger.info(f"Found {len(local_files)} files in local directory")
|
||||||
|
|
||||||
|
logger.info(f"Total files to sync: {len(files)}")
|
||||||
|
return files
|
||||||
|
|
||||||
|
def _scan_directory(self, base_path: Path, path_type: str) -> List[tuple]:
|
||||||
|
"""Scan directory for files to sync"""
|
||||||
|
files = []
|
||||||
|
|
||||||
|
for root, dirs, filenames in os.walk(base_path):
|
||||||
|
root_path = Path(root)
|
||||||
|
|
||||||
|
dirs_to_skip = []
|
||||||
|
for d in dirs:
|
||||||
|
should_skip = False
|
||||||
|
has_important_files = False
|
||||||
|
|
||||||
|
for pattern in self.exclude_patterns:
|
||||||
|
if '/' in pattern:
|
||||||
|
dir_pattern = pattern.split('/')[0]
|
||||||
|
if fnmatch.fnmatch(d, dir_pattern):
|
||||||
|
should_skip = True
|
||||||
|
break
|
||||||
|
|
||||||
|
if should_skip:
|
||||||
|
for pattern in self.include_patterns:
|
||||||
|
if '/' in pattern:
|
||||||
|
dir_pattern = pattern.split('/')[0]
|
||||||
|
if fnmatch.fnmatch(d, dir_pattern):
|
||||||
|
has_important_files = True
|
||||||
|
break
|
||||||
|
|
||||||
|
if should_skip and not has_important_files:
|
||||||
|
dirs_to_skip.append(d)
|
||||||
|
|
||||||
|
for d in dirs_to_skip:
|
||||||
|
dirs.remove(d)
|
||||||
|
|
||||||
|
for filename in filenames:
|
||||||
|
file_path = root_path / filename
|
||||||
|
if self.should_include_file(file_path, base_path):
|
||||||
|
files.append((file_path, base_path, path_type))
|
||||||
|
|
||||||
|
return files
|
||||||
|
|
||||||
|
def upload_to_s3(self, dry_run: bool = False, incremental: bool = True, cleanup: bool = False) -> bool:
|
||||||
|
"""Upload local Zen data to S3"""
|
||||||
|
files = self.get_local_files()
|
||||||
|
if not files:
|
||||||
|
logger.warning("No files found to upload")
|
||||||
|
return False
|
||||||
|
|
||||||
|
s3_objects = {}
|
||||||
|
if incremental or cleanup:
|
||||||
|
logger.info("Analyzing existing S3 objects...")
|
||||||
|
s3_objects = self._list_s3_objects()
|
||||||
|
|
||||||
|
files_to_upload, files_to_skip, files_to_delete = self._analyze_upload_files(files, s3_objects, incremental, cleanup)
|
||||||
|
|
||||||
|
self._log_sync_analysis(files_to_upload, [], files_to_skip, files_to_delete if cleanup else None)
|
||||||
|
|
||||||
|
if not files_to_upload and not files_to_delete:
|
||||||
|
logger.info("Everything is up to date!")
|
||||||
|
return True
|
||||||
|
|
||||||
|
upload_success = self._process_files(files_to_upload, "uploading", dry_run, self._upload_file_wrapper)
|
||||||
|
delete_success = True
|
||||||
|
|
||||||
|
if cleanup and files_to_delete:
|
||||||
|
delete_success = self._process_files(files_to_delete, "deleting", dry_run, self._delete_s3_file)
|
||||||
|
|
||||||
|
logger.info(f"Upload completed")
|
||||||
|
return upload_success and delete_success
|
||||||
|
|
||||||
|
def _analyze_upload_files(self, files: List, s3_objects: Dict, incremental: bool, cleanup: bool) -> Tuple[List, List, List]:
|
||||||
|
files_to_upload = []
|
||||||
|
files_to_skip = []
|
||||||
|
files_to_delete = []
|
||||||
|
|
||||||
|
logger.info(f"Analyzing {len(files)} local files...")
|
||||||
|
|
||||||
|
for file_path, base_path, path_type in files:
|
||||||
|
s3_key = self._get_s3_key(file_path, base_path, path_type)
|
||||||
|
relative_s3_key = self._get_relative_s3_key(file_path, base_path, path_type)
|
||||||
|
local_info = self._get_file_info(file_path)
|
||||||
|
|
||||||
|
if incremental and relative_s3_key in s3_objects:
|
||||||
|
s3_info = s3_objects[relative_s3_key]
|
||||||
|
if not self._files_are_different(local_info, s3_info):
|
||||||
|
files_to_skip.append((file_path, s3_key, local_info['size']))
|
||||||
|
continue
|
||||||
|
|
||||||
|
files_to_upload.append((file_path, s3_key, local_info['size'], path_type))
|
||||||
|
|
||||||
|
if cleanup:
|
||||||
|
local_s3_keys = {self._get_relative_s3_key(fp, bp, pt) for fp, bp, pt in files}
|
||||||
|
for s3_key in s3_objects:
|
||||||
|
if s3_key not in local_s3_keys:
|
||||||
|
s3_info = s3_objects[s3_key]
|
||||||
|
files_to_delete.append((s3_key, s3_info['s3_key'], s3_info['size']))
|
||||||
|
|
||||||
|
return files_to_upload, files_to_skip, files_to_delete
|
||||||
|
|
||||||
|
def download_from_s3(self, dry_run: bool = False, incremental: bool = True, cleanup: bool = False) -> bool:
|
||||||
|
"""Download Zen data from S3"""
|
||||||
|
try:
|
||||||
|
logger.info("Analyzing S3 objects...")
|
||||||
|
s3_objects = self._list_s3_objects()
|
||||||
|
|
||||||
|
if not s3_objects:
|
||||||
|
logger.warning(f"No objects found in S3 with prefix: {self.prefix}")
|
||||||
|
return False
|
||||||
|
|
||||||
|
files_to_download, files_to_skip, files_to_delete = self._analyze_download_files(s3_objects, incremental, cleanup)
|
||||||
|
|
||||||
|
self._log_sync_analysis([], files_to_download, files_to_skip, files_to_delete if cleanup else None)
|
||||||
|
|
||||||
|
if not files_to_download and not files_to_delete:
|
||||||
|
logger.info("Everything is up to date!")
|
||||||
|
return True
|
||||||
|
|
||||||
|
download_success = self._process_files(files_to_download, "downloading", dry_run, self._download_file_wrapper)
|
||||||
|
delete_success = True
|
||||||
|
|
||||||
|
if cleanup and files_to_delete:
|
||||||
|
delete_success = self._process_files(files_to_delete, "deleting local", dry_run, self._delete_local_file)
|
||||||
|
|
||||||
|
logger.info(f"Download completed")
|
||||||
|
return download_success and delete_success
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Error during download: {e}")
|
||||||
|
return False
|
||||||
|
|
||||||
|
def _analyze_download_files(self, s3_objects: Dict, incremental: bool, cleanup: bool) -> Tuple[List, List, List]:
|
||||||
|
files_to_download = []
|
||||||
|
files_to_skip = []
|
||||||
|
files_to_delete = []
|
||||||
|
|
||||||
|
logger.info(f"Analyzing {len(s3_objects)} S3 objects...")
|
||||||
|
|
||||||
|
for relative_s3_key, s3_info in s3_objects.items():
|
||||||
|
local_path = self._get_download_path(relative_s3_key)
|
||||||
|
if not local_path:
|
||||||
|
continue
|
||||||
|
|
||||||
|
local_info = self._get_file_info(local_path)
|
||||||
|
|
||||||
|
if incremental and local_info['exists']:
|
||||||
|
if not self._files_are_different(local_info, s3_info):
|
||||||
|
files_to_skip.append((local_path, s3_info['s3_key'], s3_info['size']))
|
||||||
|
continue
|
||||||
|
|
||||||
|
files_to_download.append((local_path, s3_info['s3_key'], s3_info['size'], relative_s3_key))
|
||||||
|
|
||||||
|
if cleanup:
|
||||||
|
local_files = self.get_local_files()
|
||||||
|
s3_relative_keys = set(s3_objects.keys())
|
||||||
|
|
||||||
|
for file_path, base_path, path_type in local_files:
|
||||||
|
relative_s3_key = self._get_relative_s3_key(file_path, base_path, path_type)
|
||||||
|
if relative_s3_key not in s3_relative_keys:
|
||||||
|
file_info = self._get_file_info(file_path)
|
||||||
|
if file_info['exists']:
|
||||||
|
files_to_delete.append((file_path, relative_s3_key, file_info['size']))
|
||||||
|
|
||||||
|
return files_to_download, files_to_skip, files_to_delete
|
||||||
|
|
||||||
|
def sync_bidirectional(self, dry_run: bool = False, cleanup: bool = False) -> bool:
|
||||||
|
"""Perform bidirectional sync between local and S3"""
|
||||||
|
logger.info("Starting bidirectional sync...")
|
||||||
|
|
||||||
|
local_files = self.get_local_files()
|
||||||
|
s3_objects = self._list_s3_objects()
|
||||||
|
|
||||||
|
local_lookup = {}
|
||||||
|
for file_path, base_path, path_type in local_files:
|
||||||
|
relative_s3_key = self._get_relative_s3_key(file_path, base_path, path_type)
|
||||||
|
local_lookup[relative_s3_key] = {
|
||||||
|
'path': file_path,
|
||||||
|
'info': self._get_file_info(file_path),
|
||||||
|
'path_type': path_type
|
||||||
|
}
|
||||||
|
|
||||||
|
upload_files, download_files, skip_files = self._analyze_bidirectional_sync(local_lookup, s3_objects)
|
||||||
|
|
||||||
|
self._log_sync_analysis(upload_files, download_files, skip_files)
|
||||||
|
|
||||||
|
if not upload_files and not download_files:
|
||||||
|
logger.info("Everything is in sync!")
|
||||||
|
return True
|
||||||
|
|
||||||
|
upload_success = self._process_files(upload_files, "uploading", dry_run, self._upload_file_wrapper)
|
||||||
|
download_success = self._process_files(download_files, "downloading", dry_run, self._download_file_wrapper)
|
||||||
|
|
||||||
|
logger.info("Bidirectional sync completed!")
|
||||||
|
return upload_success and download_success
|
||||||
|
|
||||||
|
def _analyze_bidirectional_sync(self, local_lookup: Dict, s3_objects: Dict) -> Tuple[List, List, List]:
|
||||||
|
upload_files = []
|
||||||
|
download_files = []
|
||||||
|
skip_files = []
|
||||||
|
|
||||||
|
for relative_key in set(local_lookup.keys()) & set(s3_objects.keys()):
|
||||||
|
local_info = local_lookup[relative_key]['info']
|
||||||
|
s3_info = s3_objects[relative_key]
|
||||||
|
|
||||||
|
if not self._files_are_different(local_info, s3_info):
|
||||||
|
skip_files.append((relative_key, None, local_info['size']))
|
||||||
|
continue
|
||||||
|
|
||||||
|
if local_info['mtime'] > s3_info['mtime']:
|
||||||
|
file_path = local_lookup[relative_key]['path']
|
||||||
|
path_type = local_lookup[relative_key]['path_type']
|
||||||
|
s3_key = s3_objects[relative_key]['s3_key']
|
||||||
|
upload_files.append((file_path, s3_key, local_info['size'], path_type))
|
||||||
|
else:
|
||||||
|
local_path = local_lookup[relative_key]['path']
|
||||||
|
s3_key = s3_objects[relative_key]['s3_key']
|
||||||
|
download_files.append((local_path, s3_key, s3_info['size'], relative_key))
|
||||||
|
|
||||||
|
for relative_key in set(local_lookup.keys()) - set(s3_objects.keys()):
|
||||||
|
local_data = local_lookup[relative_key]
|
||||||
|
file_path = local_data['path']
|
||||||
|
path_type = local_data['path_type']
|
||||||
|
|
||||||
|
base_path = self.zen_roaming_path if path_type == 'roaming' else self.zen_local_path
|
||||||
|
s3_key = self._get_s3_key(file_path, base_path, path_type)
|
||||||
|
upload_files.append((file_path, s3_key, local_data['info']['size'], path_type))
|
||||||
|
|
||||||
|
for relative_key in set(s3_objects.keys()) - set(local_lookup.keys()):
|
||||||
|
s3_info = s3_objects[relative_key]
|
||||||
|
local_path = self._get_download_path(relative_key)
|
||||||
|
if local_path:
|
||||||
|
download_files.append((local_path, s3_info['s3_key'], s3_info['size'], relative_key))
|
||||||
|
|
||||||
|
return upload_files, download_files, skip_files
|
||||||
|
|
||||||
|
def _upload_file_wrapper(self, file_path: Path, s3_key: str, size: int, path_type: str):
|
||||||
|
self._upload_file(file_path, s3_key, path_type)
|
||||||
|
|
||||||
|
def _download_file_wrapper(self, local_path: Path, s3_key: str, size: int, relative_key: str):
|
||||||
|
self._download_file(s3_key, local_path)
|
||||||
|
|
||||||
|
def _delete_s3_file(self, relative_key: str, s3_key: str, size: int):
|
||||||
|
self.s3_client.delete_object(Bucket=self.bucket, Key=s3_key)
|
||||||
|
|
||||||
|
def _delete_local_file(self, file_path: Path, relative_key: str, size: int):
|
||||||
|
file_path.unlink()
|
||||||
|
try:
|
||||||
|
file_path.parent.rmdir()
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
def _upload_file(self, file_path: Path, s3_key: str, path_type: str):
|
||||||
|
"""Upload a single file to S3"""
|
||||||
|
if not self.config.config['aws'].get('disable_metadata', False):
|
||||||
|
file_hash = calculate_file_hash(file_path)
|
||||||
|
metadata = {
|
||||||
|
'path-type': path_type,
|
||||||
|
'original-mtime': str(int(file_path.stat().st_mtime)),
|
||||||
|
'file-hash': file_hash
|
||||||
|
}
|
||||||
|
|
||||||
|
try:
|
||||||
|
with open(file_path, 'rb') as file_data:
|
||||||
|
self.s3_client.put_object(
|
||||||
|
Bucket=self.bucket,
|
||||||
|
Key=s3_key,
|
||||||
|
Body=file_data,
|
||||||
|
Metadata=metadata
|
||||||
|
)
|
||||||
|
except ClientError as e:
|
||||||
|
error_msg = str(e)
|
||||||
|
if ('AccessDenied' in error_msg or 'headers' in error_msg.lower() or
|
||||||
|
'not signed' in error_msg or 'signature' in error_msg.lower()):
|
||||||
|
logger.warning(f"Metadata error, retrying without metadata for {file_path.name}")
|
||||||
|
with open(file_path, 'rb') as file_data:
|
||||||
|
self.s3_client.put_object(
|
||||||
|
Bucket=self.bucket,
|
||||||
|
Key=s3_key,
|
||||||
|
Body=file_data
|
||||||
|
)
|
||||||
|
if not self.config.config['aws'].get('disable_metadata', False):
|
||||||
|
self.config.config['aws']['disable_metadata'] = True
|
||||||
|
self.config.save_config()
|
||||||
|
logger.info("Auto-disabled metadata for compatibility")
|
||||||
|
else:
|
||||||
|
raise
|
||||||
|
else:
|
||||||
|
with open(file_path, 'rb') as file_data:
|
||||||
|
self.s3_client.put_object(
|
||||||
|
Bucket=self.bucket,
|
||||||
|
Key=s3_key,
|
||||||
|
Body=file_data
|
||||||
|
)
|
||||||
|
|
||||||
|
def _download_file(self, s3_key: str, local_path: Path):
|
||||||
|
"""Download a single file from S3"""
|
||||||
|
local_path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
self.s3_client.download_file(
|
||||||
|
self.bucket,
|
||||||
|
s3_key,
|
||||||
|
str(local_path)
|
||||||
|
)
|
||||||
|
|
||||||
|
# Try to restore modification time
|
||||||
|
try:
|
||||||
|
obj_metadata = self.s3_client.head_object(Bucket=self.bucket, Key=s3_key)
|
||||||
|
if ('Metadata' in obj_metadata and
|
||||||
|
not self.config.config['aws'].get('disable_metadata', False)):
|
||||||
|
metadata = obj_metadata['Metadata']
|
||||||
|
original_mtime = None
|
||||||
|
if 'original-mtime' in metadata:
|
||||||
|
original_mtime = int(metadata['original-mtime'])
|
||||||
|
elif 'original_mtime' in metadata:
|
||||||
|
original_mtime = int(metadata['original_mtime'])
|
||||||
|
|
||||||
|
if original_mtime:
|
||||||
|
os.utime(local_path, (original_mtime, original_mtime))
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
def list_profiles(self) -> Dict:
|
||||||
|
"""List available Zen browser profiles"""
|
||||||
|
profiles = {}
|
||||||
|
|
||||||
|
if self.zen_roaming_path:
|
||||||
|
profiles.update(self._list_profiles_from_path(self.zen_roaming_path, "roaming"))
|
||||||
|
else:
|
||||||
|
logger.error("Roaming path not configured")
|
||||||
|
|
||||||
|
return profiles
|
||||||
|
|
||||||
|
def _list_profiles_from_path(self, zen_path: Path, path_type: str) -> Dict:
|
||||||
|
"""List profiles from a specific path"""
|
||||||
|
profiles = {}
|
||||||
|
profiles_ini = zen_path / "profiles.ini"
|
||||||
|
|
||||||
|
if not profiles_ini.exists():
|
||||||
|
logger.warning(f"profiles.ini not found in {zen_path}")
|
||||||
|
return profiles
|
||||||
|
|
||||||
|
try:
|
||||||
|
config_parser = configparser.ConfigParser()
|
||||||
|
config_parser.read(profiles_ini)
|
||||||
|
|
||||||
|
for section in config_parser.sections():
|
||||||
|
if section.startswith('Profile'):
|
||||||
|
name = config_parser.get(section, 'Name', fallback='Unknown')
|
||||||
|
path = config_parser.get(section, 'Path', fallback='')
|
||||||
|
is_default = config_parser.getboolean(section, 'Default', fallback=False)
|
||||||
|
store_id = config_parser.get(section, 'StoreID', fallback='')
|
||||||
|
|
||||||
|
profile_path = zen_path / 'Profiles' / path if path else None
|
||||||
|
|
||||||
|
profiles[section] = {
|
||||||
|
'name': name,
|
||||||
|
'path': path,
|
||||||
|
'is_default': is_default,
|
||||||
|
'store_id': store_id,
|
||||||
|
'full_path': profile_path,
|
||||||
|
'path_type': path_type,
|
||||||
|
'base_path': zen_path
|
||||||
|
}
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Error reading profiles.ini from {zen_path}: {e}")
|
||||||
|
|
||||||
|
return profiles
|
||||||
|
|
||||||
|
def get_profile_info(self) -> Dict:
|
||||||
|
"""Get comprehensive profile information"""
|
||||||
|
info = {
|
||||||
|
'system_type': 'dual-path',
|
||||||
|
'paths': {},
|
||||||
|
'profiles': {},
|
||||||
|
'profile_groups': {}
|
||||||
|
}
|
||||||
|
|
||||||
|
info['paths'] = {
|
||||||
|
'roaming': str(self.zen_roaming_path) if self.zen_roaming_path else 'Not configured',
|
||||||
|
'local': str(self.zen_local_path) if self.zen_local_path else 'Not configured',
|
||||||
|
'roaming_exists': self.zen_roaming_path.exists() if self.zen_roaming_path else False,
|
||||||
|
'local_exists': self.zen_local_path.exists() if self.zen_local_path else False
|
||||||
|
}
|
||||||
|
|
||||||
|
info['profiles'] = self.list_profiles()
|
||||||
|
|
||||||
|
if self.zen_roaming_path:
|
||||||
|
profile_groups_dir = self.zen_roaming_path / "Profile Groups"
|
||||||
|
if profile_groups_dir.exists():
|
||||||
|
info['profile_groups']['exists'] = True
|
||||||
|
info['profile_groups']['path'] = str(profile_groups_dir)
|
||||||
|
db_files = list(profile_groups_dir.glob("*.sqlite"))
|
||||||
|
info['profile_groups']['databases'] = [f.name for f in db_files]
|
||||||
|
else:
|
||||||
|
info['profile_groups']['exists'] = False
|
||||||
|
|
||||||
|
return info
|
||||||
43
utils.py
Normal file
43
utils.py
Normal file
@@ -0,0 +1,43 @@
|
|||||||
|
import hashlib
|
||||||
|
from pathlib import Path
|
||||||
|
import logging
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
def calculate_file_hash(file_path: Path, algorithm: str = 'md5') -> str:
|
||||||
|
"""Calculate hash of a file"""
|
||||||
|
if algorithm == 'md5':
|
||||||
|
hash_obj = hashlib.md5()
|
||||||
|
elif algorithm == 'sha256':
|
||||||
|
hash_obj = hashlib.sha256()
|
||||||
|
else:
|
||||||
|
raise ValueError(f"Unsupported hash algorithm: {algorithm}")
|
||||||
|
|
||||||
|
try:
|
||||||
|
with open(file_path, 'rb') as f:
|
||||||
|
for chunk in iter(lambda: f.read(8192), b""):
|
||||||
|
hash_obj.update(chunk)
|
||||||
|
return hash_obj.hexdigest()
|
||||||
|
except (OSError, IOError) as e:
|
||||||
|
logger.error(f"Error calculating hash for {file_path}: {e}")
|
||||||
|
return ""
|
||||||
|
|
||||||
|
def calculate_data_hash(data: bytes, algorithm: str = 'md5') -> str:
|
||||||
|
"""Calculate hash of data bytes"""
|
||||||
|
if algorithm == 'md5':
|
||||||
|
hash_obj = hashlib.md5()
|
||||||
|
elif algorithm == 'sha256':
|
||||||
|
hash_obj = hashlib.sha256()
|
||||||
|
else:
|
||||||
|
raise ValueError(f"Unsupported hash algorithm: {algorithm}")
|
||||||
|
|
||||||
|
hash_obj.update(data)
|
||||||
|
return hash_obj.hexdigest()
|
||||||
|
|
||||||
|
def format_size(size_bytes: int) -> str:
|
||||||
|
"""Format file size in human readable format"""
|
||||||
|
for unit in ['B', 'KB', 'MB', 'GB']:
|
||||||
|
if size_bytes < 1024.0:
|
||||||
|
return f"{size_bytes:.1f}{unit}"
|
||||||
|
size_bytes /= 1024.0
|
||||||
|
return f"{size_bytes:.1f}TB"
|
||||||
16
zensync.py
Normal file
16
zensync.py
Normal file
@@ -0,0 +1,16 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
import logging
|
||||||
|
from cli import run_cli
|
||||||
|
|
||||||
|
logging.basicConfig(
|
||||||
|
level=logging.INFO,
|
||||||
|
format='%(asctime)s - %(levelname)s - %(message)s',
|
||||||
|
datefmt='%Y-%m-%d %H:%M:%S'
|
||||||
|
)
|
||||||
|
|
||||||
|
def main():
|
||||||
|
"""Main entry point"""
|
||||||
|
run_cli()
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
Reference in New Issue
Block a user