wip(22w9a): add item zipper

src/__init__.py

@@ -1,8 +1,6 @@
 from dotenv import load_dotenv
-
-from .glob import file_exists
 from .db import Database, dbname
-from .fs import FileSystem
+from .fs import FileSystem, file_exists
 from .backup import Backup
 
 if not file_exists(".env"):

src/backup.py

@@ -1,7 +1,6 @@
-import os
-import importlib
 from typing import Union
 
+from .cloud import Storage as StorageClient
 from . import Database, FileSystem
 from . import dbname
 
@@ -12,18 +11,9 @@ class Backup(FileSystem):
         self.has_change = False
 
         self.db = Database()
-        self._cloud = None
+        self.cloud = StorageClient()
 
-        self.cloud = os.getenv("SERVICE")
-
-    @property
-    def cloud(self):
-        return self._cloud
-
-    @cloud.setter
-    def cloud(self, service: str):
-        CloudClient = importlib.import_module("." + service, "Client")
-        self._cloud = CloudClient()
+        self.zip = self.db.get_flag("ZIP")
 
     # Backup a file or folder
     def backup_item(self, item: Union[list, str]) -> bool:
@@ -44,8 +34,13 @@ class Backup(FileSystem):
 
         print(f"Uploading: '{item[0]}' ... ", end="")
 
+        blob = item
+        # Upload as zip archive
+        if self.zip:
+            blob = FileSystem.zip(blob)
+
         # Upload to cloud
-        if self.cloud.upload(item):
+        if self.cloud.upload(blob):
             # Update local database
             if self.db.set_item(item):
                 print("OK")

src/cloud/__init__.py

@@ -1 +1,30 @@
-from .gcs import client as GoogleCloudStorage
+import os
+import importlib
+
+# This class initializes only the module for the requested service.
+# It sits as an intermediate between the initiator script and client library.
+class Storage:
+    def __init__(self):
+        self._service = None
+        self.service = os.getenv("PROVIDER_NAME")
+
+    @property
+    def service(self):
+        return self._service
+
+    # Create a new storage client for the requested service
+    @service.setter
+    def service(self, service: str):
+        if not service:
+            service = "gcs"
+        module = importlib.import_module("src.cloud." + service)
+
+        self._service = module.StorageClient(os.getenv("TARGET_BUCKET"))
+
+    @staticmethod
+    def get_args(values):
+        values.pop(-1)
+        return values
+
+    def upload(self, *argv):
+        return self.service.upload(*argv)

src/cloud/gcs.py

@@ -0,0 +1,10 @@
+from google.cloud import storage
+
+# Client for Google Cloud Storage
+class StorageClient:
+    def __init__(self, bucket):
+        client = storage.Client()
+        self.bucket = client.bucket(bucket)
+
+    def upload(self, item):
+        blob = self.bucket.blob()

src/cloud/gcs/__init__.py

@@ -1 +0,0 @@
-from .client import StorageClient

src/cloud/gcs/client.py

@@ -1,21 +0,0 @@
-import os
-from os.path import exists
-from google.cloud import storage
-
-class StorageClient(storage.Client):
-    def __init__(self, bucket: str = None):
-        if not bucket:
-            bucket = os.getenv("TARGET_BUCKET")
-        
-        if not self.gcloud_key_exists():
-            raise Exception("GOOGLE_APPLICATION_CREDENTIALS has to point to a key file")
-
-        super().__init__()
-
-    # Check if env var is set to a key file
-    def gcloud_key_exists(self) -> bool:
-        keyfile = os.getenv("GOOGLE_APPLICATION_CREDENTIALS")
-
-        if not keyfile or not exists(keyfile):
-            return False
-        return True

src/db/database.py

@@ -1,3 +1,4 @@
+import os
 from typing import Union
 
 from .sqlite import SQLite
@@ -34,10 +35,10 @@ class Database(SQLite):
     # Check if item should be backed up by comparing mtime and checksum
     def check_item(self, item: Union[list, tuple]) -> bool:
         sql = f"SELECT {self.columns} FROM manifest WHERE anchor = '{item[0]}'"
-        db_item = self.query(sql)[0]
+        db_item = self.query(sql)
 
         # New item or item changed, so back it up
-        if not db_item or (item != db_item):
+        if not db_item or (item != db_item[0]):
             return True
         return False
 

src/db/flags.py

@@ -0,0 +1,16 @@
+from .sqlite import SQLite
+
+class Flags(SQLite):
+    def __init__(self):
+        super().__init__()
+
+        self._columns = ["k", "v"]
+
+    @property
+    def columns(self):
+        return ",".join(self._columns)
+
+    @columns.setter
+    def columns(self, columns: list):
+        self._columns = columns
+

src/db/sqlite.py

@@ -2,8 +2,6 @@ import os
 import pathlib
 import sqlite3 as sqlite
 
-from ..glob import file_exists
-
 dbname = "._cloudbackup.db"
 
 class SQLite():
@@ -13,7 +11,7 @@ class SQLite():
 
         # Check if the database requires configuration
         try:
-            db_exists = self.query("SELECT k FROM flags WHERE k = 'INIT'")
+            db_exists = self.get_flag("INIT")
             if not db_exists:
                 self.configure_db()
         except sqlite.OperationalError:
@@ -55,3 +53,17 @@ class SQLite():
         sql_str = SQLite.format_query(sql.read())
 
         return self.cursor.executescript(sql_str)
+
+        # Get value from flag by key or .env override
+    def get_flag(self, key: str) -> bool:
+        # Return environment variable override
+        envar = os.getenv(key)
+        if envar:
+            return envar
+
+        sql = f"SELECT v FROM flags WHERE k = '{key}'"
+        res = self.query(sql)
+
+        if not res:
+            return False
+        return True

src/fs/__init__.py

@@ -1 +1,2 @@
+from .utils import file_exists, get_parent, get_file
 from .fs import FileSystem

src/fs/fs.py

@@ -1,7 +1,10 @@
 import os
 import zlib
+import shutil
+import tempfile
 
 from ..db import dbname
+from .utils import file_exists, get_parent, get_file
 
 class FileSystem:
     def __init__(self):
@@ -17,6 +20,15 @@ class FileSystem:
         encoded = data.encode("utf-8")
         return zlib.crc32(encoded)
 
+    @staticmethod
+    def zip(item) -> str:
+        dest = f"{tempfile.gettempdir()}/{str(item[1])}"
+
+        # Make a temp zip file of single file or folder
+        if file_exists(item[0]):
+            return shutil.make_archive(dest, "zip", get_parent(item[0]), get_file(item[0]))
+        return shutil.make_archive(dest, "zip", item[0])
+
     # Get metadata from candidate file or folder
     def get_item(self, path: str) -> tuple:
         # Ignore SQLite temp files

src/fs/utils.py

@@ -0,0 +1,15 @@
+import os.path
+import ntpath
+
+# Check if a file exists
+def file_exists(file: str) -> bool:
+    return os.path.isfile(file)
+
+# Get parent directory of file
+def get_parent(path: str) -> str:
+    return os.path.dirname(path)
+
+# Get filename from path string
+def get_file(path: str) -> str:
+    head, tail = ntpath.split(path)
+    return tail or ntpath.basename(head)

src/glob.py

@@ -1,4 +0,0 @@
-import os.path
-
-def file_exists(file: str) -> bool:
-    return os.path.isfile(file)