.example.config.json

@@ -1,16 +1,26 @@
-[
 {
+	"config": {
+		"cloud": {
+			"bucket": "my-bucket"
+		},
+		"archive": {
 			"password": "mypassword",
-		"compression": 10,
+			"compression_level": 10,
-		"abspath_temp": null,
+			"default_path_temp": "/tmp/"
-		"abspath_target": "<replace with ABSOLUTE path to a target directory>",
+		}
-		"abspath_destination": "s3://<replace with bucket>/<replace with destination>"
+	},
+	"archive": [
+		{
+			"compress": true,
+			"path_temp": null,
+			"path_target_to": "<replace with bucket object path>",
+			"path_target_from": "<replace with ABSOLUTE path to a target directory>"
 		},
 		{
-		"password": "mypassword",
+			"compress": true,
-		"compression": 10,
+			"path_temp": null,
-		"abspath_temp": null,
+			"path_target_to": "<replace with bucket object path>",
-		"abspath_target": "<replace with ABSOLUTE path to a target directory>",
+			"path_target_from": "<replace with ABSOLUTE path to a target directory>"
-		"abspath_destination": "s3://<replace with bucket>/<replace with destination>"
 		}
 	]
+}

README.md

@@ -1,14 +1,17 @@
 # 3rd
-A script to automate the 3rd "off-site copy" step in the 3-2-1 Backup strategy. Each directory have independent configuration of compression level, encryption password, AWS S3 destination, and temporary storage location while being uploaded to S3. 
+A script to automate the "off-site copy" in the 3-2-1 Backup strategy with encryption, uploading to AWS S3, and independent definition of compression method and [temporary] archive storage locations for uploading large archives to S3, with support for independent configurations for subdirectories as well.
 
 This script is a wrapper for the AWS CLI `aws` and the 7zip CLI `7z` and is meant to be run on Linux. Other operating systems are untested.
 
+## Key features
+- Archive encryption before uploading to AWS S3.
+- Independent compression level, archive location, S3 storage location, for directories and subdirectories.
+
 # Installation
 Make sure you have the following prerequisites before starting:
-- Python 3
+- Python 3 installed.
-- The [7zip CLI](https://www.7-zip.org/download.html)
+- The [7zip CLI](https://www.7-zip.org/download.html) installed.
-- The [AWS CLI](https://aws.amazon.com/cli/)
+- The [AWS CLI](https://aws.amazon.com/cli/) installed and configured with write access to your target bucket.
-- Write permission to an AWS S3 bucket
 
 
 1. **Clone this repository**
@@ -27,10 +30,10 @@ cp -p .example.config.json .config.json
 
 [See the config file documentation for more information](#config).
 
-4. **Run `run.py` with your config file**
+4. **Run `run.py` in autorun mode**
 
 ```
-python3 run.py -i .config.json
+python3 run.py -a
 ```
 [See the CLI section for a list of all available arguments](#cli).
 
@@ -38,26 +41,49 @@ python3 run.py -i .config.json
 ## Optional cron
 Schedule this backup script to run with a crontab entry, for example:
 ```
-30 2 * * 3 cd /opt/3rd && /usr/bin/python3 run.py -i .config.json
+30 2 * * 3 cd /opt/3rd && /usr/bin/python3 run.py -a
 ```
 Which will run at 2:30 each Wednesday.
 
 # Config
-The config file (`.config.json` by default) is used to define which directories to archive with parameters.
+The config file `.config.json` is used to define parameters and which directories to archive (in autorun mode).
-
-Directories are specified as an array of objects, each object has key value properties with the following format:
 
 ```json
-[
 {
-		"password": "mypassword", // AES-256 encryption password. Set to false to disable encryption
+	"config": {
-		"compression": 10, // Compression level between 0-10, where 0 is STORE and 10 is max compression. Set to 0 or false/null to disable compression
+		"cloud": {
-		"abspath_temp": "/tmp", // Directory to store the created archive while it's being uploaded to S3. Set to false/null to use the system temp-directory
+            // Name of the target AWS S3 bucket
-		"abspath_target": "<replace with ABSOLUTE path to a target directory>", // An ABSOLUTE path to the directory or file to archive
+			"bucket": "vlw-test"
-		"abspath_destination": "s3://<replace with bucket>/<replace with destination>" // A fully qualified AWS S3 URL
+            // .. More options to come (probably)
 		},
-    // etc..
+        // Default settings for each archive item
-]
+		"archive": {
+            // The password used to encrypt all archives
+			"password": "mypassword",
+            // The compression level to use when "compress" is true for an item
+			"compression_level": 10,
+            // Default archive location when "path_temp" is null for an item
+			"default_path_temp": "/tmp/output"
+		}
+	},
+    // Array of archive items, see next section
+	"archive": []
+}
+
+```
+
+Each archive item uses the following structure:
+```json
+{
+    // Enables or disables compression for this directory. STORE will be used if disabled.
+    "compress": true,
+    // Store the encrypted archive in this directory temporarily while its being uploaded to S3.
+    "path_temp": "/tmp/",
+    // The relative path from the bucket root directory to store the uploaded object
+    "path_target_to": "/myarchive.7z",
+    // An absolute path (very important) to the target folder to upload
+    "path_target_from": "/my/archive"
+}
 ```
 
 ## Common parent directories
@@ -66,21 +92,19 @@ One of the key features of this script is that it can perform different archivin
 If you have the directory `/my/archive` with the following config:
 ```json
 {
-	"password": "mypassword",
+    "compress": true,
-	"compression": 10,
+    "path_temp": null,
-	"abspath_temp": null,
+    "path_target_to": "/myarchive.7z",
-	"abspath_target": "/my/archive",
+    "path_target_from": "/my/archive"
-	"abspath_destination": "s3://my-bucket/archive.7z"
 }
 ```
 And a subdirectory `/my/archive/subdirectory` with the following config:
 ```json
 {
-	"password": "mypassword",
+    "compress": true,
-	"compression": 10,
+    "path_temp": null,
-	"abspath_temp": null,
+    "path_target_to": "/my-subdirectory.7z",
-	"abspath_target": "/my/archive/subdirectory",
+    "path_target_from": "/my/archive/subdirectory"
-	"abspath_destination": "s3://my-bucket/subdirectory.7z"
 }
 ```
 The `/my/archive/subdirectory` will be **excluded** from the `/my/archive` archive since it has an overriding archive configuration.
@@ -91,6 +115,6 @@ Available command line argument with `run.py`:
 arg|Name|Default|Description
 --|--|--|--
 `-s`|`--sleep`|2|Set a global sleep duration between commands
-`-i`|`--input`|*None*|Path to a config file to load
+`-a`|`--autorun`|False|Archive each item in the .config.json archive array
 `-d`|`--dryrun`|False|Perform a dry run. Archives will not be uploaded to S3.
 `-l`|`--log-level`|`StdoutLevel.STANDARD`|Set a custom log level when printing to the console. See `/src/Enums.py#StdoutLevel`

run.py

@@ -1,4 +1,3 @@
-import os
 import typing
 import argparse
 
@@ -10,18 +9,21 @@ from src.Enums import StdoutLevel, Namespace
 
 stdout = Stdout(Namespace.CLI)
 
-def main() -> None:
+def autorun() -> None:
 	"""
-	Autorun from a config file
+	Autorun
-
-	Args:
-		file (str): Path to the config file to load
 	"""
 
+	for item in Config().config["archive"]:
+		stdout.info(f"Autorun: {item}")
+
+		Aws(Archive(item)).upload()
+
+if __name__ == "__main__":
 	parser = argparse.ArgumentParser(description="Testing")
 
 	parser.add_argument("-s", "--sleep", type=int, help="Global log sleep level")
-	parser.add_argument("-i", "--input", help="Load config file from path",default=".config.json")
+	parser.add_argument("-a", "--autorun", action="store_true", help="Autorun")
 	parser.add_argument("-d", "--dryrun", action="store_true", help="Dry run")
 	parser.add_argument("-l", "--log-level", type=str, help="Global log level")
 
@@ -43,19 +45,12 @@ def main() -> None:
 	# Set enable dry run
 	if args.dryrun:
 		Aws.dry_run = True
-		Archive.preserve_archives = True
 		stdout.ok("Dry run enabled")
 
 	stdout.log("Starting...")
 
-	if not os.path.isfile(args.input):
+	# Autorun archives from config
-		stdout.error(f"No config file found at path: '{args.input}'")
+	if args.autorun:
-		exit(1)
+		autorun()
-
-	for item in Config.from_json_file(args.input):
-		Aws(Archive(item)).upload()
 
 	stdout.log("Finished!")
-
-if __name__ == "__main__":
-	main()

src/Archive/Archive.py

@@ -6,23 +6,21 @@ import subprocess
 from ..Cli import Cli
 from ..Stdout import Stdout
 from ..Config import Config
-from .Filesystem import Filesystem
+from .Filesystem import PATH_MANIFEST, Filesystem
 from ..Enums import Namespace, Format, StdoutLevel
 
 class Archive():
-	preserve_archives = False
+	def __init__(self, item: dict):
-
-	def __init__(self, item: Config):
 		"""
 		Create a new Archive instance for a target item
 
 		Args:
-			item (Config): Target item to archive
+			item (dict): A dictionary of archive instructions
 		"""
 
 		self.item = item
-
+		self.__fs = Filesystem(self.item["path_target_from"])
-		self.__fs = Filesystem(self.item.abspath_target)
+		self.__config = Config().config["config"]
 		self.__stdout = Stdout(Namespace.ARCHIVE)
 
 		if self.__fs.valid:
@@ -39,18 +37,19 @@ class Archive():
 			str: Absolute pathname to target zip file
 		"""
 
-		filename = hashlib.md5(self.item.abspath_target.encode()).hexdigest()
+		output_path = self.__config["archive"]["default_path_temp"]
 
-		return f"{self.item.abspath_temp.rstrip('/')}/{filename}.7z"
+		# Override temporary file location if specified
+		if self.item["path_temp"]:
+			output_path = self.item["path_temp"]
+
+		return f"{output_path.rstrip('/')}/{hashlib.md5(self.item['path_target_from'].encode()).hexdigest()}.7z"
 
 	def cleanup(self) -> None:
 		"""
 		Remove archive file
 		"""
 
-		if Archive.preserve_archives:
-			return
-
 		os.remove(self.output_path)
 		self.__stdout.info(f"Archive removed: {self.output_path}")
 
@@ -59,33 +58,31 @@ class Archive():
 		Skip archiving of target item
 		"""
 
-		self.__stdout.warn(f"Archiving skipped for: {self.item.abspath_target}")
+		self.__stdout.warn(f"Archiving skipped for: {self.item['path_target_from']}")
-
-		self.cleanup()
 
 	def __compress(self) -> None:
 		"""
 		Compress the target path
 		"""
 
-		self.__stdout.log(f"Starting compression for: {self.item.abspath_target}").sleep()
+		self.__stdout.log(f"Starting compression for: {self.item['path_target_from']}").sleep()
 
 		# Prepare command line arguments
 		args = [
 			"7z",
 			"a",
 			"-t7z",
-			f"-mx={self.item.compression}"
+			f"-mx={self.__config['archive']['compression_level']}"
 		]
 
 		# Enable encryption if archive password is set
-		if self.item.password:
+		if self.__config["archive"]["password"]:
 			args.append("-mhe=on")
-			args.append(f"-p{self.item.password}")
+			args.append(f"-p{self.__config['archive']['password']}")
 
 		# Append output path and file list manifest arguments for 7zip
 		args.append(self.output_path)
-		args.append(self.item.abspath_target)
+		args.append(self.item["path_target_from"])
 
 		# Exclude directories thats
 		for exclude in self.__fs.common_relative_paths():
@@ -99,7 +96,6 @@ class Archive():
 			return self.__die()
 
 		self.__stdout.info(f"Temporary archive placed at: {self.__fs.path}").sleep()
-		self.__stdout.ok(f"Compression completed for: {self.item.abspath_target}")
+		self.__stdout.ok(f"Compression completed for: {self.item['path_target_from']}")
 
 		cmd.cleanup()
-		self.cleanup()

src/Archive/Filesystem.py

@@ -1,10 +1,13 @@
 import os
+import tempfile
 from typing import Union
 
 from ..Config import Config
 from ..Stdout import Stdout
 from ..Enums import Namespace
 
+PATH_MANIFEST = f"{tempfile.gettempdir().rstrip('/')}/archive_manifest.txt"
+
 class Filesystem():
 	def __init__(self, path: str):
 		"""
@@ -16,6 +19,7 @@ class Filesystem():
 
 		self.valid = True
 		self.path = path
+		self.__config = Config().config
 		self.__stdout = Stdout(Namespace.FILESYSTEM)
 
 		if not os.path.exists(self.path):
@@ -33,8 +37,8 @@ class Filesystem():
 
 		paths = []
 
-		for path in Config.pathnames:
+		for item in self.__config["archive"]:
-			paths.append(path)
+			paths.append(item["path_target_from"])
 
 		return paths
 
@@ -71,7 +75,6 @@ class Filesystem():
 		Returns:
 			str | None: Common pathname with base path or None if no common path (or is base path)
 		"""
-		
 		base_path = os.path.normpath(self.path)
 		target_path = os.path.normpath(path)
 

src/Config.py

@@ -1,174 +1,25 @@
 import json
-import tempfile
+import typing
 from pathlib import Path
-from typing import Self, Union
 
-from .Enums import ConfigKeys
+CONFIG_FILEPATH = Path.cwd() / ".config.json"
 
 class Config():
-	pathnames = set()
+	def __init__(self):
-
-	@staticmethod
-	def for_each(items: list) -> Self:
-		"""
-		Returns a generator which iterates over each item in a list of item configs
-
-		Args:
-			items (list): The list to iterate over
-
-		Returns:
-			Self: Instance of the Config class
-
-		Yields:
-			Iterator[Self]: Config class for the current item
-		"""
-
-		for item in items:
-			yield Config(item)
-
-	@staticmethod
-	def from_json_file(pathname: str) -> Self:
-		"""
-		Load item configs from a JSON file
-
-		Args:
-			pathname (str): _description_
-
-		Returns:
-			Self: _description_
-		"""
-
-		with open(pathname, "r") as f:
-			config = json.load(f)
-
-			for item in config:
-				Config.pathnames.add(item[ConfigKeys.ABSPATH_TARGET.value])
-
-			return Config.for_each(config)
-
-	@staticmethod
-	def __throw_missing_key(key: ConfigKeys) -> None:
-		"""
-		Raises a KeyError for an item config key if it does not exist
-
-		Args:
-			key (ConfigKeys): The key to raise an error for
-
-		Raises:
-			KeyError: Raised from an item config key
-		"""
-
-		raise KeyError(f"Expected required item config key '{key.value}' but it was not found")
-
-	@staticmethod
-	def __throw_value_error(key: ConfigKeys, expected_type: str) -> None:
-		"""
-		Raise a ValueError for a key with an expected type
-
-		Args:
-			key (ConfigKeys): The item config key to raise an error for
-			expected_type (str): The expected type
-
-		Raises:
-			ValueError: Raised from an item config key and expected value type
-		"""
-
-		raise ValueError(f"Item config key '{key.value}' expects a value of type {expected_type}")
-
-	def __init__(self, item: dict):
 		"""
 		Create a new Config instance
 		"""
 
-		self.__item = item
+		with open(CONFIG_FILEPATH, "r") as f:
+			self.__config = json.load(f)
 
 	@property
-	def password(self) -> str|bool:
+	def config(self) -> dict:
 		"""
-		Returns the password for this item, or None if unset
+		Returns config variables as a dictonary
 
 		Returns:
-			str|False: Password or None if no password is set
+			dict: Confg values
 		"""
 		
-		if not self.__key_exists(ConfigKeys.PASSWORD.value):
+		return dict(self.__config)
-			return False
-
-		return self.__item[ConfigKeys.PASSWORD.value] if isinstance(self.__item[ConfigKeys.PASSWORD.value], str) else None
-
-	@property
-	def compression(self) -> int:
-		"""
-		Returns the compression level for this item, or false if STORE mode should be used
-
-		Returns:
-			str|False: Compression level for this item, false if compression is disabled
-		"""
-
-		if not self.__key_exists(ConfigKeys.COMPRESSION.value):
-			return 0
-
-		if not isinstance(self.__item[ConfigKeys.COMPRESSION.value], int) or self.__item[ConfigKeys.COMPRESSION.value] == 0:
-			return 0
-
-		return max(1, min(self.__item[ConfigKeys.COMPRESSION.value], 10))
-
-	@property
-	def abspath_temp(self) -> str:
-		"""
-		Returns the path to the directory where the created archive will be stored until it's uploaded
-
-		Returns:
-			str: Absolute path to the destination directory
-		"""
-
-		if not self.__key_exists(ConfigKeys.ABSPATH_TEMP.value):
-			return tempfile.gettempdir()
-
-		return self.__item[ConfigKeys.ABSPATH_TEMP.value] if isinstance(self.__item[ConfigKeys.ABSPATH_TEMP.value], str) else tempfile.gettempdir()
-
-	@property
-	def abspath_target(self) -> str:
-		"""
-		Returns an absolute path to the target to be archived
-
-		Returns:
-			str: Absolute path to the target
-		"""
-
-		if not self.__key_exists(ConfigKeys.ABSPATH_TARGET.value):
-			return Config.__throw_missing_key(ConfigKeys.ABSPATH_TARGET)
-
-		if not isinstance(self.__item[ConfigKeys.ABSPATH_TARGET.value], str):
-			return Config.__throw_value_error(ConfigKeys.ABSPATH_TARGET, str)
-
-		return self.__item[ConfigKeys.ABSPATH_TARGET.value]
-
-	@property
-	def abspath_destination(self) -> str:
-		"""
-		Returns an absolute path to the target to be archived
-
-		Returns:
-			str: Absolute path to the target
-		"""
-
-		if not self.__key_exists(ConfigKeys.ASBPATH_DESTINATION.value):
-			return Config.__throw_missing_key(ConfigKeys.ASBPATH_DESTINATION)
-
-		if not isinstance(self.__item[ConfigKeys.ASBPATH_DESTINATION.value], str):
-			return Config.__throw_value_error(ConfigKeys.ASBPATH_DESTINATION, str)
-
-		return self.__item[ConfigKeys.ASBPATH_DESTINATION.value]
-
-	def __key_exists(self, key: str) -> bool:
-		"""
-		Returns true if a property key is defined for the current item
-
-		Args:
-			key (str): The key to test
-
-		Returns:
-			bool: True if key exists
-		"""
-		return key in self.__item

src/Enums.py

@@ -1,12 +1,5 @@
 from enum import Enum
 
-class ConfigKeys(Enum):
-	PASSWORD            = "password"
-	COMPRESSION         = "compression"
-	ABSPATH_TEMP        = "abspath_temp"
-	ABSPATH_TARGET      = "abspath_target"
-	ASBPATH_DESTINATION = "abspath_destination"
-
 class Namespace(Enum):
 	AWS        = "AWS"
 	CLI        = "Command"

src/Stdout.py

@@ -103,7 +103,7 @@ class Stdout():
 		"""
 
 		# Bail out if stdout is disabled
-		if self.global_level.value == StdoutLevel.NONE.value:
+		if Stdout.global_level.value == StdoutLevel.NONE.value:
 			return self.die()
 
 		print(f"{Format.HEADER.value}> {self.namespace.value}:{Format.ENDC.value}{msg}{Format.ENDC.value}")
@@ -136,7 +136,7 @@ class Stdout():
 		"""
 
 		# Bail out if log level is less than verbose
-		if not self.global_level.value >= StdoutLevel.STANDARD.value:
+		if not Stdout.global_level.value >= StdoutLevel.STANDARD.value:
 			return self.die()
 
 		return self.print(f" {msg}")
@@ -153,7 +153,7 @@ class Stdout():
 		"""
 
 		# Bail out if log level is less than verbose
-		if not self.global_level.value >= StdoutLevel.VERBOSE.value:
+		if not Stdout.global_level.value >= StdoutLevel.VERBOSE.value:
 			return self.die()
 
 		return self.print(f" {msg}")
@@ -170,7 +170,7 @@ class Stdout():
 		"""
 
 		# Bail out if log level is less than verbose
-		if not self.global_level.value >= StdoutLevel.DEBUG.value:
+		if not Stdout.global_level.value >= StdoutLevel.DEBUG.value:
 			return self.die()
 
 		return self.print(f" {msg}")
@@ -187,7 +187,7 @@ class Stdout():
 		"""
 
 		# Bail out if log level is less than default
-		if not self.global_level.value >= StdoutLevel.STANDARD.value:
+		if not Stdout.global_level.value >= StdoutLevel.STANDARD.value:
 			return self.die()
 
 		return self.print(f" {Format.WARNING.value}! WARN: {msg}")

src/Upload/Aws.py

@@ -1,6 +1,7 @@
 import typing
 
 from ..Cli import Cli
+from ..Config import Config
 from ..Stdout import Stdout
 from ..Enums import Namespace, StdoutLevel
 from ..Archive.Archive import Archive
@@ -17,6 +18,7 @@ class Aws():
 		"""
 
 		self.archive = archive
+		self.__config = Config().config
 		self.__stdout = Stdout(Namespace.AWS)
 
 	def upload(self) -> None:
@@ -24,16 +26,15 @@ class Aws():
 		Create a backup of an Archive instance to AWS
 		"""
 
-		self.__stdout.log(f"Starting upload of archive for: {self.archive.item.abspath_target}")
+		self.__stdout.log(f"Starting upload of archive for: {self.archive.item['path_target_from']}")
 		self.__stdout.debug(f"Archive object: {self.archive}")
-		self.__stdout.info(f"Uploading to: {self.archive.item.abspath_destination}")
 
 		args = [
 			"aws",
 			"s3",
 			"cp",
 			self.archive.output_path,
-			self.archive.item.abspath_destination
+			f"s3://{self.__config['config']['cloud']['bucket']}/{self.archive.item['path_target_to'].strip('/')}"
 		]
 
 		if Aws.dry_run:
@@ -46,7 +47,7 @@ class Aws():
 		cmd.run(args)
 
 		if cmd.stderr:
-			self.__stdout.error(f"Failed to upload archive for: {self.archive.item.abspath_target}")
+			self.__stdout.error(f"Failed to upload archive for: {self.archive.item['path_target_from']}")
 			return
 
 		self.__stdout.info("Cleaning up temporary files")
@@ -54,4 +55,4 @@ class Aws():
 		cmd.cleanup()
 		self.archive.cleanup()
 
-		self.__stdout.ok(f"Archive uploaded to: {self.archive.item.abspath_destination}")
+		self.__stdout.ok(f"Archive uploaded: {self.archive.item['path_target_from']}")