diff --git a/.env.example b/.env.example index 47ab5eb..ac20780 100644 --- a/.env.example +++ b/.env.example @@ -1,9 +1,9 @@ # Path to the local folder to back up -SOURCE_FOLDER= +SOURCE_FOLDER="" # Name of the remote bucket (destination) -TARGET_BUCKET= +TARGET_BUCKET="" # Cloud provider (gcs, s3, azure) -SERVICE_NAME= -# Path to service account key file -SERVICE_KEY= \ No newline at end of file +SERVICE_NAME="" +# Cloud provider access string or path to key file +SERVICE_KEY="" \ No newline at end of file diff --git a/install.sh b/install.sh new file mode 100755 index 0000000..91d324e --- /dev/null +++ b/install.sh @@ -0,0 +1,28 @@ +install () { + python3 -m pip install $1 +} + +install python-dotenv + +# Install Python libraries for cloud provider +case $1 in + "gcs") + install google-cloud-storage + ;; + + "azure") + install azure-storage-blob + ;; + + "aws") + install boto3 + ;; + + *) ;; +esac + +# Create .env file if it doesn't exist +if [ ! -f ".env" ]; then + cp .env.example .env + sed -i "s/SERVICE_NAME=\"\"/SERVICE_NAME=\"$1\"" .env +fi \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index dd9601f..a053fe2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,4 @@ python-dotenv -google-cloud-storage \ No newline at end of file +google-cloud-storage +azure-storage-blob +boto3 \ No newline at end of file diff --git a/src/cloud/azure.py b/src/cloud/azure.py new file mode 100644 index 0000000..cc939fe --- /dev/null +++ b/src/cloud/azure.py @@ -0,0 +1,31 @@ +import os +from azure.storage.blob import BlobServiceClient + +from ..fs.utils import get_file + +class StorageClient: + def __init__(self): + self.client = BlobServiceClient.from_connection_string(os.getenv("SERVICE_KEY")) + + self._error = None + + @property + def error(self): + return self._error + + @error.setter + def error(self, state): + self._error = state + + def upload(self, path: str) -> bool: + name = get_file(path) + blob = self.client.get_blob_client(container=os.getenv("TARGET_BUCKET"), blob=name) + + try: + with open(path, "rb") as f: + blob.upload_blob(f,overwrite=True) + return True + except Exception as e: + if e.response.status_code == 403: + self.error = "Account lacks 'storage.objects.create' permissions on this bucket " + return False \ No newline at end of file diff --git a/src/db/database.py b/src/db/database.py index c808e0f..a7ea58d 100644 --- a/src/db/database.py +++ b/src/db/database.py @@ -28,14 +28,17 @@ class Database(SQLite): # Check if item exists in the database def item_exists(self, item: Union[list, tuple]) -> bool: sql = "SELECT anchor FROM manifest WHERE anchor = ?" - res = self.query(sql, (item[0])) + res = self.query(sql, (item[0],)) return res # Check if item should be backed up by comparing mtime and checksum def check_item(self, item: Union[list, tuple]) -> bool: + if os.getenv("FORCE_UPLOAD"): + return True + sql = f"SELECT {self.columns} FROM manifest WHERE anchor = ?" - db_item = self.query(sql, (item[0])) + db_item = self.query(sql, (item[0],)) # New item or item changed, so back it up if not db_item or (item != db_item[0]): @@ -44,10 +47,12 @@ class Database(SQLite): # Insert or update item in database def set_item(self, item: Union[list, tuple]) -> bool: + values = [item[0], item[1], item[0]] sql = "UPDATE manifest SET anchor = ?, chksum = ? WHERE anchor = ?" if not self.item_exists(item): sql = f"INSERT INTO manifest ({self.columns}) VALUES (?, ?)" - self.query(sql, (item[0], item[1], item[0])) + values.pop() + self.query(sql, values) return True \ No newline at end of file