From bc9ba0783fbe90a393d43bf7f39daf83da7bbd53 Mon Sep 17 00:00:00 2001 From: Cloud Shell Date: Tue, 1 Mar 2022 15:52:38 +0000 Subject: [PATCH] wip(22w9a): add item zipper --- src/__init__.py | 4 +--- src/backup.py | 23 +++++++++-------------- src/cloud/__init__.py | 31 ++++++++++++++++++++++++++++++- src/cloud/gcs.py | 10 ++++++++++ src/cloud/gcs/__init__.py | 1 - src/cloud/gcs/client.py | 21 --------------------- src/db/database.py | 5 +++-- src/db/flags.py | 16 ++++++++++++++++ src/db/sqlite.py | 18 +++++++++++++++--- src/fs/__init__.py | 1 + src/fs/fs.py | 12 ++++++++++++ src/fs/utils.py | 15 +++++++++++++++ src/glob.py | 4 ---- 13 files changed, 112 insertions(+), 49 deletions(-) create mode 100644 src/cloud/gcs.py delete mode 100644 src/cloud/gcs/__init__.py delete mode 100644 src/cloud/gcs/client.py create mode 100644 src/db/flags.py create mode 100644 src/fs/utils.py delete mode 100644 src/glob.py diff --git a/src/__init__.py b/src/__init__.py index 079a1a0..efa8b99 100644 --- a/src/__init__.py +++ b/src/__init__.py @@ -1,8 +1,6 @@ from dotenv import load_dotenv - -from .glob import file_exists from .db import Database, dbname -from .fs import FileSystem +from .fs import FileSystem, file_exists from .backup import Backup if not file_exists(".env"): diff --git a/src/backup.py b/src/backup.py index 121e251..2206eb1 100644 --- a/src/backup.py +++ b/src/backup.py @@ -1,7 +1,6 @@ -import os -import importlib from typing import Union +from .cloud import Storage as StorageClient from . import Database, FileSystem from . import dbname @@ -12,18 +11,9 @@ class Backup(FileSystem): self.has_change = False self.db = Database() - self._cloud = None - - self.cloud = os.getenv("SERVICE") + self.cloud = StorageClient() - @property - def cloud(self): - return self._cloud - - @cloud.setter - def cloud(self, service: str): - CloudClient = importlib.import_module("." + service, "Client") - self._cloud = CloudClient() + self.zip = self.db.get_flag("ZIP") # Backup a file or folder def backup_item(self, item: Union[list, str]) -> bool: @@ -44,8 +34,13 @@ class Backup(FileSystem): print(f"Uploading: '{item[0]}' ... ", end="") + blob = item + # Upload as zip archive + if self.zip: + blob = FileSystem.zip(blob) + # Upload to cloud - if self.cloud.upload(item): + if self.cloud.upload(blob): # Update local database if self.db.set_item(item): print("OK") diff --git a/src/cloud/__init__.py b/src/cloud/__init__.py index 2a02487..fe8d569 100644 --- a/src/cloud/__init__.py +++ b/src/cloud/__init__.py @@ -1 +1,30 @@ -from .gcs import client as GoogleCloudStorage \ No newline at end of file +import os +import importlib + +# This class initializes only the module for the requested service. +# It sits as an intermediate between the initiator script and client library. +class Storage: + def __init__(self): + self._service = None + self.service = os.getenv("PROVIDER_NAME") + + @property + def service(self): + return self._service + + # Create a new storage client for the requested service + @service.setter + def service(self, service: str): + if not service: + service = "gcs" + module = importlib.import_module("src.cloud." + service) + + self._service = module.StorageClient(os.getenv("TARGET_BUCKET")) + + @staticmethod + def get_args(values): + values.pop(-1) + return values + + def upload(self, *argv): + return self.service.upload(*argv) \ No newline at end of file diff --git a/src/cloud/gcs.py b/src/cloud/gcs.py new file mode 100644 index 0000000..992e481 --- /dev/null +++ b/src/cloud/gcs.py @@ -0,0 +1,10 @@ +from google.cloud import storage + +# Client for Google Cloud Storage +class StorageClient: + def __init__(self, bucket): + client = storage.Client() + self.bucket = client.bucket(bucket) + + def upload(self, item): + blob = self.bucket.blob() \ No newline at end of file diff --git a/src/cloud/gcs/__init__.py b/src/cloud/gcs/__init__.py deleted file mode 100644 index 94841ed..0000000 --- a/src/cloud/gcs/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .client import StorageClient \ No newline at end of file diff --git a/src/cloud/gcs/client.py b/src/cloud/gcs/client.py deleted file mode 100644 index 2a29b9b..0000000 --- a/src/cloud/gcs/client.py +++ /dev/null @@ -1,21 +0,0 @@ -import os -from os.path import exists -from google.cloud import storage - -class StorageClient(storage.Client): - def __init__(self, bucket: str = None): - if not bucket: - bucket = os.getenv("TARGET_BUCKET") - - if not self.gcloud_key_exists(): - raise Exception("GOOGLE_APPLICATION_CREDENTIALS has to point to a key file") - - super().__init__() - - # Check if env var is set to a key file - def gcloud_key_exists(self) -> bool: - keyfile = os.getenv("GOOGLE_APPLICATION_CREDENTIALS") - - if not keyfile or not exists(keyfile): - return False - return True \ No newline at end of file diff --git a/src/db/database.py b/src/db/database.py index 7a09d64..8eebf39 100644 --- a/src/db/database.py +++ b/src/db/database.py @@ -1,3 +1,4 @@ +import os from typing import Union from .sqlite import SQLite @@ -34,10 +35,10 @@ class Database(SQLite): # Check if item should be backed up by comparing mtime and checksum def check_item(self, item: Union[list, tuple]) -> bool: sql = f"SELECT {self.columns} FROM manifest WHERE anchor = '{item[0]}'" - db_item = self.query(sql)[0] + db_item = self.query(sql) # New item or item changed, so back it up - if not db_item or (item != db_item): + if not db_item or (item != db_item[0]): return True return False diff --git a/src/db/flags.py b/src/db/flags.py new file mode 100644 index 0000000..a555a3e --- /dev/null +++ b/src/db/flags.py @@ -0,0 +1,16 @@ +from .sqlite import SQLite + +class Flags(SQLite): + def __init__(self): + super().__init__() + + self._columns = ["k", "v"] + + @property + def columns(self): + return ",".join(self._columns) + + @columns.setter + def columns(self, columns: list): + self._columns = columns + diff --git a/src/db/sqlite.py b/src/db/sqlite.py index a0ed6b5..d15e1a4 100644 --- a/src/db/sqlite.py +++ b/src/db/sqlite.py @@ -2,8 +2,6 @@ import os import pathlib import sqlite3 as sqlite -from ..glob import file_exists - dbname = "._cloudbackup.db" class SQLite(): @@ -13,7 +11,7 @@ class SQLite(): # Check if the database requires configuration try: - db_exists = self.query("SELECT k FROM flags WHERE k = 'INIT'") + db_exists = self.get_flag("INIT") if not db_exists: self.configure_db() except sqlite.OperationalError: @@ -55,3 +53,17 @@ class SQLite(): sql_str = SQLite.format_query(sql.read()) return self.cursor.executescript(sql_str) + + # Get value from flag by key or .env override + def get_flag(self, key: str) -> bool: + # Return environment variable override + envar = os.getenv(key) + if envar: + return envar + + sql = f"SELECT v FROM flags WHERE k = '{key}'" + res = self.query(sql) + + if not res: + return False + return True diff --git a/src/fs/__init__.py b/src/fs/__init__.py index 91c2a7b..49dfed6 100644 --- a/src/fs/__init__.py +++ b/src/fs/__init__.py @@ -1 +1,2 @@ +from .utils import file_exists, get_parent, get_file from .fs import FileSystem \ No newline at end of file diff --git a/src/fs/fs.py b/src/fs/fs.py index 0ecb555..3f7e664 100644 --- a/src/fs/fs.py +++ b/src/fs/fs.py @@ -1,7 +1,10 @@ import os import zlib +import shutil +import tempfile from ..db import dbname +from .utils import file_exists, get_parent, get_file class FileSystem: def __init__(self): @@ -17,6 +20,15 @@ class FileSystem: encoded = data.encode("utf-8") return zlib.crc32(encoded) + @staticmethod + def zip(item) -> str: + dest = f"{tempfile.gettempdir()}/{str(item[1])}" + + # Make a temp zip file of single file or folder + if file_exists(item[0]): + return shutil.make_archive(dest, "zip", get_parent(item[0]), get_file(item[0])) + return shutil.make_archive(dest, "zip", item[0]) + # Get metadata from candidate file or folder def get_item(self, path: str) -> tuple: # Ignore SQLite temp files diff --git a/src/fs/utils.py b/src/fs/utils.py new file mode 100644 index 0000000..62d6531 --- /dev/null +++ b/src/fs/utils.py @@ -0,0 +1,15 @@ +import os.path +import ntpath + +# Check if a file exists +def file_exists(file: str) -> bool: + return os.path.isfile(file) + +# Get parent directory of file +def get_parent(path: str) -> str: + return os.path.dirname(path) + +# Get filename from path string +def get_file(path: str) -> str: + head, tail = ntpath.split(path) + return tail or ntpath.basename(head) \ No newline at end of file diff --git a/src/glob.py b/src/glob.py deleted file mode 100644 index 28d3b19..0000000 --- a/src/glob.py +++ /dev/null @@ -1,4 +0,0 @@ -import os.path - -def file_exists(file: str) -> bool: - return os.path.isfile(file) \ No newline at end of file