From fd67f4f7f7d1bc70b792276700626459fbb0951f Mon Sep 17 00:00:00 2001 From: Cloud Shell Date: Mon, 28 Feb 2022 14:41:33 +0000 Subject: [PATCH] wip(22w8a): finish db writer --- .env.example | 5 ++++- backup.py | 3 +-- src/__init__.py | 2 +- src/backup.py | 51 ++++++++++++++++++++++++++++++++++++++++----- src/db/config.sql | 3 +-- src/db/database.py | 52 +++++++++++++++++++++++++++++++++------------- src/db/sqlite.py | 5 +++-- src/fs/fs.py | 18 ++++++++++++---- 8 files changed, 107 insertions(+), 32 deletions(-) diff --git a/.env.example b/.env.example index 9a2cede..74dc8b6 100644 --- a/.env.example +++ b/.env.example @@ -1,4 +1,7 @@ SOURCE_FOLDER= TARGET_BUCKET= -GOOGLE_APPLICATION_CREDENTIALS= \ No newline at end of file +# Cloud provider "gcs, aws, azure" +PROVIDER_NAME= +# Path to service account key file +PROVIDER_KEY= \ No newline at end of file diff --git a/backup.py b/backup.py index f8266ee..31763ab 100644 --- a/backup.py +++ b/backup.py @@ -2,5 +2,4 @@ import sys from src import Backup -Backup().backup_all() -print("OK") \ No newline at end of file +Backup().backup_all() \ No newline at end of file diff --git a/src/__init__.py b/src/__init__.py index 5c467e0..079a1a0 100644 --- a/src/__init__.py +++ b/src/__init__.py @@ -1,7 +1,7 @@ from dotenv import load_dotenv from .glob import file_exists -from .db import Database +from .db import Database, dbname from .fs import FileSystem from .backup import Backup diff --git a/src/backup.py b/src/backup.py index ac2d259..121e251 100644 --- a/src/backup.py +++ b/src/backup.py @@ -1,12 +1,31 @@ +import os +import importlib from typing import Union from . import Database, FileSystem +from . import dbname class Backup(FileSystem): def __init__(self): super().__init__() - self.db = Database() + self.has_change = False + + self.db = Database() + self._cloud = None + + self.cloud = os.getenv("SERVICE") + + @property + def cloud(self): + return self._cloud + + @cloud.setter + def cloud(self, service: str): + CloudClient = importlib.import_module("." + service, "Client") + self._cloud = CloudClient() + + # Backup a file or folder def backup_item(self, item: Union[list, str]) -> bool: if isinstance(item, str): item = self.get_item(item) @@ -14,12 +33,34 @@ class Backup(FileSystem): # Check item against db if it has changed db_resp = self.db.check_item(item) if not db_resp: - return True + return - self.db.set_item(item) + # Back up changes to database in silence + if item[0].endswith(dbname): + self.db.set_item(item) + return + + self.has_change = True + + print(f"Uploading: '{item[0]}' ... ", end="") + + # Upload to cloud + if self.cloud.upload(item): + # Update local database + if self.db.set_item(item): + print("OK") + else: + print("OK, BUT: Failed to update database") + else: + print("FAILED") - return True + return + # Scan TARGET_FOLDER for files and folders to back up def backup_all(self): + # Check all second-level files and folder at target path for item in self.all(): - self.backup_item(item) \ No newline at end of file + self.backup_item(item) + + if not self.has_change: + print("Up to date. No changes found") \ No newline at end of file diff --git a/src/db/config.sql b/src/db/config.sql index e0d4e63..23bb830 100644 --- a/src/db/config.sql +++ b/src/db/config.sql @@ -5,8 +5,7 @@ CREATE TABLE flags ( CREATE TABLE manifest ( anchor TEXT PRIMARY KEY, - mtime INTEGER, - chksum TEXT + chksum INTEGER ); INSERT INTO flags diff --git a/src/db/database.py b/src/db/database.py index 9b61ff9..7a09d64 100644 --- a/src/db/database.py +++ b/src/db/database.py @@ -1,30 +1,52 @@ +from typing import Union + from .sqlite import SQLite class Database(SQLite): def __init__(self): super().__init__() - def item_exists(self, item: list) -> bool: + self._columns = ["anchor", "chksum"] + + @property + def columns(self): + return ",".join(self._columns) + + @columns.setter + def columns(self, columns: list): + self._columns = columns + + # Create SQL string CSV from list + @staticmethod + def str_csv(items: Union[list, tuple]) -> str: + items = list(map(lambda value : f"'{str(value)}'", items)) + items = ",".join(items) + + return items + + # Check if item exists in the database + def item_exists(self, item: Union[list, tuple]) -> bool: sql = f"SELECT anchor FROM manifest WHERE anchor = '{item[0]}'" res = self.query(sql) return res - # Test if a candidate item should be backed up - def check_item(self, item: list) -> bool: - sql = f"SELECT anchor, mtime, chksum FROM manifest WHERE anchor = '{item[0]}'" - res = self.query(sql) - if not res: - return True + # Check if item should be backed up by comparing mtime and checksum + def check_item(self, item: Union[list, tuple]) -> bool: + sql = f"SELECT {self.columns} FROM manifest WHERE anchor = '{item[0]}'" + db_item = self.query(sql)[0] - return res + # New item or item changed, so back it up + if not db_item or (item != db_item): + return True + return False # Insert or update item in database - def set_item(self, item: list) -> bool: - values = ",".join(item) - sql = f"INSERT INTO manifest (anchor, mtime, chksum) VALUES ({values})" - if not self.item_exists(item): - sql = f"UPDATE manifest SET anchor = '{item[0]}', mtime = '{item[1]}', chksum = '{item[2]}' WHERE anchor = '{item[0]}'" - res = self.query(sql) + def set_item(self, item: Union[list, tuple]) -> bool: + sql = f"UPDATE manifest SET anchor = '{item[0]}', chksum = {item[1]} WHERE anchor = '{item[0]}'" - return res \ No newline at end of file + if not self.item_exists(item): + sql = f"INSERT INTO manifest ({self.columns}) VALUES ('{item[0]}', {item[1]})" + self.query(sql) + + return True \ No newline at end of file diff --git a/src/db/sqlite.py b/src/db/sqlite.py index 5a40b8b..a0ed6b5 100644 --- a/src/db/sqlite.py +++ b/src/db/sqlite.py @@ -4,7 +4,7 @@ import sqlite3 as sqlite from ..glob import file_exists -dbname = ".cloudbackup.db" +dbname = "._cloudbackup.db" class SQLite(): def __init__(self): @@ -27,8 +27,9 @@ class SQLite(): # Run SQL query def query(self, sql: str): query = self.cursor.execute(sql) - result = query.fetchall() + self.db.commit() + result = query.fetchall() if len(result) < 1: return False diff --git a/src/fs/fs.py b/src/fs/fs.py index da74572..0ecb555 100644 --- a/src/fs/fs.py +++ b/src/fs/fs.py @@ -18,15 +18,25 @@ class FileSystem: return zlib.crc32(encoded) # Get metadata from candidate file or folder - def get_item(self, path: list) -> list: + def get_item(self, path: str) -> tuple: + # Ignore SQLite temp files + if path.endswith(".db-journal"): + return False + mtime = os.path.getmtime(path) chksum = FileSystem.chksum(path + str(mtime)) - data = [path, mtime, chksum] + data = (path, chksum) return data # Get all second-level files and folders for path def all(self) -> list: content = [os.path.join(self.path, f) for f in os.listdir(self.path)] - content = list(map(self.get_item, content)) - return content \ No newline at end of file + items = [] + + for item in content: + data = self.get_item(item) + if data: + items.append(data) + + return items \ No newline at end of file