wip(22w8a): finish db writer

.env.example

@@ -1,4 +1,7 @@
 SOURCE_FOLDER=
 TARGET_BUCKET=
 
-GOOGLE_APPLICATION_CREDENTIALS=
+# Cloud provider "gcs, aws, azure"
+PROVIDER_NAME=
+# Path to service account key file
+PROVIDER_KEY=

backup.py

@@ -3,4 +3,3 @@ import sys
 from src import Backup
 
 Backup().backup_all()
-print("OK")

src/__init__.py

@@ -1,7 +1,7 @@
 from dotenv import load_dotenv
 
 from .glob import file_exists
-from .db import Database
+from .db import Database, dbname
 from .fs import FileSystem
 from .backup import Backup
 

src/backup.py

@@ -1,12 +1,31 @@
+import os
+import importlib
 from typing import Union
 
 from . import Database, FileSystem
+from . import dbname
 
 class Backup(FileSystem):
     def __init__(self):
         super().__init__()
-        self.db = Database()
 
+        self.has_change = False
+
+        self.db = Database()
+        self._cloud = None
+        
+        self.cloud = os.getenv("SERVICE")
+
+    @property
+    def cloud(self):
+        return self._cloud
+
+    @cloud.setter
+    def cloud(self, service: str):
+        CloudClient = importlib.import_module("." + service, "Client")
+        self._cloud = CloudClient()
+
+    # Backup a file or folder
     def backup_item(self, item: Union[list, str]) -> bool:
         if isinstance(item, str):
             item = self.get_item(item)
@@ -14,12 +33,34 @@ class Backup(FileSystem):
         # Check item against db if it has changed
         db_resp = self.db.check_item(item)
         if not db_resp:
-            return True
+            return
 
-        self.db.set_item(item)
+        # Back up changes to database in silence
+        if item[0].endswith(dbname):
+            self.db.set_item(item)
+            return
 
-        return True
+        self.has_change = True
 
+        print(f"Uploading: '{item[0]}' ... ", end="")
+
+        # Upload to cloud
+        if self.cloud.upload(item):
+            # Update local database
+            if self.db.set_item(item):
+                print("OK")
+            else:
+                print("OK, BUT: Failed to update database")
+        else:
+            print("FAILED")
+        
+        return
+
+    # Scan TARGET_FOLDER for files and folders to back up
     def backup_all(self):
+        # Check all second-level files and folder at target path
         for item in self.all():
             self.backup_item(item)
+        
+        if not self.has_change:
+            print("Up to date. No changes found")

src/db/config.sql

@@ -5,8 +5,7 @@ CREATE TABLE flags (
 
 CREATE TABLE manifest (
     anchor TEXT PRIMARY KEY,
-    mtime INTEGER,
+    chksum INTEGER
-    chksum TEXT
 );
 
 INSERT INTO flags

src/db/database.py

@@ -1,30 +1,52 @@
+from typing import Union
+
 from .sqlite import SQLite
 
 class Database(SQLite):
     def __init__(self):
         super().__init__()
 
-    def item_exists(self, item: list) -> bool:
+        self._columns = ["anchor", "chksum"]
+
+    @property
+    def columns(self):
+        return ",".join(self._columns)
+
+    @columns.setter
+    def columns(self, columns: list):
+        self._columns = columns
+
+    # Create SQL string CSV from list
+    @staticmethod
+    def str_csv(items: Union[list, tuple]) -> str:
+        items = list(map(lambda value : f"'{str(value)}'", items))
+        items = ",".join(items)
+
+        return items
+
+    # Check if item exists in the database
+    def item_exists(self, item: Union[list, tuple]) -> bool:
         sql = f"SELECT anchor FROM manifest WHERE anchor = '{item[0]}'"
         res = self.query(sql)
 
         return res
 
-    # Test if a candidate item should be backed up
+    # Check if item should be backed up by comparing mtime and checksum
-    def check_item(self, item: list) -> bool:
+    def check_item(self, item: Union[list, tuple]) -> bool:
-        sql = f"SELECT anchor, mtime, chksum FROM manifest WHERE anchor = '{item[0]}'"
+        sql = f"SELECT {self.columns} FROM manifest WHERE anchor = '{item[0]}'"
-        res = self.query(sql)
+        db_item = self.query(sql)[0]
-        if not res:
-            return True
 
-        return res
+        # New item or item changed, so back it up
+        if not db_item or (item != db_item):
+            return True
+        return False
 
     # Insert or update item in database
-    def set_item(self, item: list) -> bool:
+    def set_item(self, item: Union[list, tuple]) -> bool:
-        values = ",".join(item)
+        sql = f"UPDATE manifest SET anchor = '{item[0]}', chksum = {item[1]} WHERE anchor = '{item[0]}'"
-        sql = f"INSERT INTO manifest (anchor, mtime, chksum) VALUES ({values})"
-        if not self.item_exists(item):
-            sql = f"UPDATE manifest SET anchor = '{item[0]}', mtime = '{item[1]}', chksum = '{item[2]}' WHERE anchor = '{item[0]}'"
-        res = self.query(sql)
 
-        return res
+        if not self.item_exists(item):
+            sql = f"INSERT INTO manifest ({self.columns}) VALUES ('{item[0]}', {item[1]})"
+        self.query(sql)
+
+        return True

src/db/sqlite.py

@@ -4,7 +4,7 @@ import sqlite3 as sqlite
 
 from ..glob import file_exists
 
-dbname = ".cloudbackup.db"
+dbname = "._cloudbackup.db"
 
 class SQLite():
     def __init__(self):
@@ -27,8 +27,9 @@ class SQLite():
     # Run SQL query
     def query(self, sql: str):
         query = self.cursor.execute(sql)
-        result = query.fetchall()
+        self.db.commit()
 
+        result = query.fetchall()
         if len(result) < 1:
             return False
         

src/fs/fs.py

@@ -18,15 +18,25 @@ class FileSystem:
         return zlib.crc32(encoded)
 
     # Get metadata from candidate file or folder
-    def get_item(self, path: list) -> list:
+    def get_item(self, path: str) -> tuple:
+        # Ignore SQLite temp files
+        if path.endswith(".db-journal"):
+            return False
+        
         mtime = os.path.getmtime(path)
         chksum = FileSystem.chksum(path + str(mtime))
 
-        data = [path, mtime, chksum]
+        data = (path, chksum)
         return data
 
     # Get all second-level files and folders for path
     def all(self) -> list:
         content = [os.path.join(self.path, f) for f in os.listdir(self.path)]
-        content = list(map(self.get_item, content))
+        items = []
-        return content
+
+        for item in content:
+            data = self.get_item(item)
+            if data:
+                items.append(data)
+
+        return items