From edc3b6fb9d59a69f172cd6f544ef9c9d6ac98d14 Mon Sep 17 00:00:00 2001
From: Cloud Shell <cloud-shell@victor-westerlund.iam.gserviceaccount.com>
Date: Fri, 25 Feb 2022 01:04:00 +0000
Subject: [PATCH] wip(22w8a): add wip db and gcs client

---
 .env.example                                |   4 ++
 .gitignore                                  |  50 ++++++++++++++++++++
 .theia/launch.json                          |  14 ++++++
 backup.py                                   |   4 ++
 requirements.txt                            |   2 +
 src/__init__.py                             |  10 ++++
 src/__pycache__/__init__.cpython-39.pyc     | Bin 0 -> 429 bytes
 src/__pycache__/glob.cpython-39.pyc         | Bin 0 -> 328 bytes
 src/db/__init__.py                          |   1 +
 src/db/__pycache__/__init__.cpython-39.pyc  | Bin 0 -> 182 bytes
 src/db/__pycache__/sqlite.cpython-39.pyc    | Bin 0 -> 1454 bytes
 src/db/sqlite.py                            |  43 +++++++++++++++++
 src/gcs/__init__.py                         |   1 +
 src/gcs/__pycache__/__init__.cpython-39.pyc | Bin 0 -> 190 bytes
 src/gcs/__pycache__/client.cpython-39.pyc   | Bin 0 -> 1008 bytes
 src/gcs/client.py                           |  21 ++++++++
 src/glob.py                                 |   4 ++
 17 files changed, 154 insertions(+)
 create mode 100644 .env.example
 create mode 100644 .gitignore
 create mode 100644 .theia/launch.json
 create mode 100644 backup.py
 create mode 100644 requirements.txt
 create mode 100644 src/__init__.py
 create mode 100644 src/__pycache__/__init__.cpython-39.pyc
 create mode 100644 src/__pycache__/glob.cpython-39.pyc
 create mode 100644 src/db/__init__.py
 create mode 100644 src/db/__pycache__/__init__.cpython-39.pyc
 create mode 100644 src/db/__pycache__/sqlite.cpython-39.pyc
 create mode 100644 src/db/sqlite.py
 create mode 100644 src/gcs/__init__.py
 create mode 100644 src/gcs/__pycache__/__init__.cpython-39.pyc
 create mode 100644 src/gcs/__pycache__/client.cpython-39.pyc
 create mode 100644 src/gcs/client.py
 create mode 100644 src/glob.py

diff --git a/.env.example b/.env.example
new file mode 100644
index 0000000..9a2cede
--- /dev/null
+++ b/.env.example
@@ -0,0 +1,4 @@
+SOURCE_FOLDER=
+TARGET_BUCKET=
+
+GOOGLE_APPLICATION_CREDENTIALS=
\ No newline at end of file
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..5d9c0c8
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,50 @@
+# Bootstrapping #
+#################
+/node_modules
+/public/hot
+/public/storage
+/storage/*.key
+/vendor
+.env
+.env.backup
+.phpunit.result.cache
+Homestead.json
+Homestead.yaml
+npm-debug.log
+yarn-error.log
+public/robots.txt
+
+
+# OS generated files #
+######################
+.DS_Store
+.DS_Store?
+._*
+.Spotlight-V100
+.Trashes
+Icon?
+ehthumbs.db
+Thumbs.db
+.directory
+
+# Tool specific files #
+#######################
+# vim
+*~
+*.swp
+*.swo
+# sublime text & textmate
+*.sublime-*
+*.stTheme.cache
+*.tmlanguage.cache
+*.tmPreferences.cache
+# Eclipse
+.settings/*
+# JetBrains, aka PHPStorm, IntelliJ IDEA
+.idea/*
+# NetBeans
+nbproject/*
+# Visual Studio Code
+.vscode
+# Sass preprocessor
+.sass-cache/
\ No newline at end of file
diff --git a/.theia/launch.json b/.theia/launch.json
new file mode 100644
index 0000000..6077c7f
--- /dev/null
+++ b/.theia/launch.json
@@ -0,0 +1,14 @@
+{
+  // Use IntelliSense to learn about possible attributes.
+  // Hover to view descriptions of existing attributes.
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "name": "Python: Current File",
+      "type": "python",
+      "request": "launch",
+      "program": "${file}",
+      "console": "integratedTerminal"
+    }
+  ]
+}
diff --git a/backup.py b/backup.py
new file mode 100644
index 0000000..dbbf772
--- /dev/null
+++ b/backup.py
@@ -0,0 +1,4 @@
+from src import SQLite, StorageClient
+
+client = SQLite()
+print("OK")
\ No newline at end of file
diff --git a/requirements.txt b/requirements.txt
new file mode 100644
index 0000000..dd9601f
--- /dev/null
+++ b/requirements.txt
@@ -0,0 +1,2 @@
+python-dotenv
+google-cloud-storage
\ No newline at end of file
diff --git a/src/__init__.py b/src/__init__.py
new file mode 100644
index 0000000..f76f657
--- /dev/null
+++ b/src/__init__.py
@@ -0,0 +1,10 @@
+from dotenv import load_dotenv
+
+from .glob import file_exists
+from .db import SQLite
+from .gcs import StorageClient
+
+if not file_exists(".env"):
+	raise FileNotFoundError("Environment variable file does not exist. Copy '.env.example' to '.env'")
+
+load_dotenv()
\ No newline at end of file
diff --git a/src/__pycache__/__init__.cpython-39.pyc b/src/__pycache__/__init__.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..70a95df52d2fc4d01db2db4d7cf0cd17d75f9b7a
GIT binary patch
literal 429
zcmYjNJ#WG=5Vi9~XkqBq(VY-Olc8%>At9(#RY(<EmMDp@V9Ab=Z6aWVU(>(D%EVvj
z)Js@;mY?38-#wpQ42O3F<M{Mg`jn7MoBR(ClL>D7fxro;k|>%{O1Q^;=_x<MHspZ}
zRG5Vw4`rl!S&tI%d32(70x|EM=#c_F5mdxWGVWi|kBsU0Uh0D9Tsx?nD^AeWzJ-Jw
zcEUP~l4$w$DI6dhE}b^T3Z_y(?G8bLO^5ec-3X&=g(cP$Miiw4)><&G!LnLA)_Emt
zsyBN!YP%BH6>1}4#GJlWMvHMji*AFM_SspmlDafOOFdqiuqFPgvL-@yewgD83+?85
zTl1MQ+WZg`<4Fg<FX>t<NE=b%$aw?Sfsv?8(crl-)mk)=T2rNYE^6WOJlX8e@vG8&
UE8(sE2s=O+cms;di)rlr0Roh8v;Y7A

literal 0
HcmV?d00001

diff --git a/src/__pycache__/glob.cpython-39.pyc b/src/__pycache__/glob.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..e670f8dd93d6af4789249cb903c7a847bb7a83df
GIT binary patch
literal 328
zcmYjMJx{|h6to?;4IkYZSQs<3{sRahv3Kc~rAlJcq;l)(JtqQ*5&i`OjQl09O#B5V
z-isJ|(!1xgPj_-PpPviD+s#$Irv1(06d8$o8oVa5!b&en>6FW?dRI2H`c;?O%C#sa
zKjaXrLM8p!d6%Z(Vh@K#^7TLX%!v;)cu5q+PJXBt@yQ8tRb)v8O!`>GHcdO^phAKo
z=a+yPtzTo>uy<zYniya+IE)T_f3RlT;G%-2?FMHM8ngAGULK#*1&3_hQ-=|`8#zr#
qI3q45<t*Tm>p+uE8AeFD4#AJ_W913u?8$HU!S>$W@o&boT>Swy?MOQS

literal 0
HcmV?d00001

diff --git a/src/db/__init__.py b/src/db/__init__.py
new file mode 100644
index 0000000..cff90f1
--- /dev/null
+++ b/src/db/__init__.py
@@ -0,0 +1 @@
+from .sqlite import SQLite
\ No newline at end of file
diff --git a/src/db/__pycache__/__init__.cpython-39.pyc b/src/db/__pycache__/__init__.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..d6a8f13ec8e6ef95545d443864918376fed3455a
GIT binary patch
literal 182
zcmYe~<>g`kf;K6MBq<>M7{oyaOhAqU5Elyoi4=wu#vF!R#wbQch7_h?22JLdj6h*c
z##?N`fj*ffseYPFQEbJ9IY4F+Nasq1B4!{3CVrXeXXNLm>X&6Em*f}4m!%d173GxX
zrRb+87waY#C1+%orRo<KCF`do>Bq-s=4F<|$LkeT-r}&y%}*)KNwovn^%;m60L1Gm
AiU0rr

literal 0
HcmV?d00001

diff --git a/src/db/__pycache__/sqlite.cpython-39.pyc b/src/db/__pycache__/sqlite.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..1f14c77c99fb35c60b4d2ef2967587eb585ea23f
GIT binary patch
literal 1454
zcmZ`(&2Aev5GJ|Y)xT^cMbQLBp;Mg&66uft$)PD)IC64uWz<ShU@ydK$yiEOyEe&n
zVnI2@kCB6P>?`q_lV2g1c4k)&6r<sS!=E8%zHd0Qai`NH(4u=kOm;aTpK!6=JXrh&
z-Sk07B56vZYNQd9OnRv|=MksmIgwoYmqhv>SmaAOCSh;|cgX09fefi@y_%&;sCP4C
zO@f;3_QlT&*i|H=l7M|G*(HfQ#ijR=miBq%E3N_=d}OjF>z6#LfwlqKrfk8gF59vL
zcN($_2zFdB{_}8VRYJF%;T*V<edy*M2uoIUNmgu4)-VchNnU#^z9ul|u<x%41<qiP
zj>5nN<}3x1u9oCkrV{H~C&~&r5vQ@8IwmJBND6Io?f5L4SzykMN?-j$bQ^qqel(co
zb2T`hB{tXMTp6o$T4Zu?l9&gvPNuVSH846E$jPAc)IYs&bs^B05H$oHfFN|2w%KB*
zBG|vygF6qdB^dPUhQY;N68;G$KoBX@HGR*P>@@`X4bgWc8<QpU&>LN0j1EFiH(-Pi
zzf(zJRaDpc+axbC3z`;a^bQCYXl07jmh>vo!EOJnQ2GL=rVfGt+9u^}(cT~~)gg0S
zLJZJ|Ep+y6+_%7lB}*#l2ezVXg2(|Y8#x-ImHbp?&QH?ZsB3h$;fnKZUL@0-;2Mcm
zu~qPF-@_q;=|E_gE_OHIzNka0zXRP@3G`JwpMX}<_Y{*zvMQo0hOsq|mQWyfpfKKe
z`aaD0;}Z3fRwJPIM~iMB9)9!geL0Cb<D<Wh_lM%(=<wO_`08J1pXng<9LtRhAk|cM
z?&>O&=IzW*H&h+Z;<<7hQ}6~=jTv(ny6J)-l+&iyWNli9Z*glwcPoRx%@kh%nL<gx
zJ0Mfc7|52za6CL5?!W9Ih~B~R(I34kbHzNy^zQxrVt734*^5*4bkD|<RPFVCA3ZCN
zKi!)v8_U?ndoLl@=wrhvYF7UW0k&LF&2MtUpV!aOHx&Wg_@cGpg!M8Cn^-!CO9<B#
zVxG$a{zAgFg*YqXw9;q{4b+X*B>=}Kd7kP!uvtb=qnj<)SjQR@TVvi}F&aDp5e>SH
vUEXb%`(dDegx#oC#p0p%VdeNqnol%lP<iP8th)Ly70h3;3QKBWnzh+~D$zuC

literal 0
HcmV?d00001

diff --git a/src/db/sqlite.py b/src/db/sqlite.py
new file mode 100644
index 0000000..7669091
--- /dev/null
+++ b/src/db/sqlite.py
@@ -0,0 +1,43 @@
+import os
+import sqlite3 as sqlite
+
+from ..glob import file_exists
+
+class SQLite():
+    def __init__(self):
+        self.db = sqlite.connect(self.get_db_path())
+        self.cursor = self.db.cursor()
+
+        self.init()
+
+    def query(self, sql: str):
+        result = self.cursor.execute(sql)
+
+        if result.rowcount < 1:
+            return False
+        
+        return result
+
+    def get_db_path(self) -> str:
+        name = ".gcsarchive.db"
+        path = os.getenv("SOURCE_FOLDER")
+
+        # Append db file name if absent
+        if not path.endswith(name):
+            # Append tailing slash if absent
+            if path[-1] != "/":
+                path += "/"
+            path += name
+        return path
+
+    def configure_db(self):
+        metadata_sql = "CREATE TABLE metadata (key text, value text)"
+        metadata = self.query(metadata_sql)
+
+    def init(self):
+        # Set up db if it's fresh
+        hasmeta_sql = "SELECT name FROM sqlite_master WHERE type='table' AND name='metadata'"
+        if not self.query(hasmeta_sql):
+            self.configure_db()
+
+        return True
\ No newline at end of file
diff --git a/src/gcs/__init__.py b/src/gcs/__init__.py
new file mode 100644
index 0000000..94841ed
--- /dev/null
+++ b/src/gcs/__init__.py
@@ -0,0 +1 @@
+from .client import StorageClient
\ No newline at end of file
diff --git a/src/gcs/__pycache__/__init__.cpython-39.pyc b/src/gcs/__pycache__/__init__.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..51fe10b7de75d290eb9868e67b98d7407b7eb631
GIT binary patch
literal 190
zcmYe~<>g`k0!C4ZBt;<o7{oyaOhAqU5Elyoi4=wu#vF!R#wbQch7_h?22JLdj6h*c
z##_9>CHX~(>8Z{+nW=dtews{CY{_6|5y*g*3`NX93QYVm*U!k$P1P^UOa^L+FH0>h
zNiE7L%}dcwPcGI?EK1JEEKAidE=mUR_2c6+^D;}~<Mj$EZ*kb<=BJeAq}qY({0zhl
E03$&ybpQYW

literal 0
HcmV?d00001

diff --git a/src/gcs/__pycache__/client.cpython-39.pyc b/src/gcs/__pycache__/client.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..fd7a831bb427e837eea8397dd08d1e86a00cf97d
GIT binary patch
literal 1008
zcmaJ=O=}x55Y=cu<J$dz651YIUl&>%3M~{$sq4BXG&mu4FMAQ2m11ur@4Av~l8|2N
zK>kDy$uWOP*Pi+pdaAT`f<vJL8ok*;nt4yNNv&2v#)79$_U9O(Up81b*90HTl5dQo
zhyq8k-7$v=jxkv5aA)GiE}$NwF7?h3^_ct3TR5>#VSpn4w>gXY5on(sWl~9l0;vj-
zj96+@>*lQj`N{x>VxTC-G-M9N7jTXafI8GYtHmz!sCNO>r@@&U`{&3-abSvV=-Oa;
zb%$pxS7q7#S$fD+Y6#w%#m;KU7Y3YyysywDP`H90aEdPR6_Ryj>Mr0XD0hJ_y(?r1
zn<`E}t_|CJz3z~F{Mz}_9iF`H?d<fnyQIClyS>$E54U#uq_fxk)a?(q+S`N1IFSui
zG^RzCD+`jwAv<myWSm95#)Z`Wh$)uO^`lYBiy0**O_p-$jqXv(rYb9P?aA4c2^|uW
z<(VQR!rGOL9|)`4HRugOQl3aj$Tj-&x-~8)tToS4qX?NZshHrioVG@(e3^)JoXuHF
ziqyufw3MqkJr?)ORlZrW91U=xf<p-L$>tqlo3|~PcNI*{M4o~<bz?UIcgzP_H*QT_
z5St30!UE(oiU($bjHGTquK!=d&%?+OwlNV{XA4AFRLvF#MlGwQcwjT7duUN;oCVSX
zWQbG^os;KxiTopY1pO$q8t2J`5uz)EObR;V7FP-RK1=v2<47e+y|lYu6uhjbYpXX%
z8RvwowNx`bh_WyHTiF+r7KHG)A}ol4lAaC<*_<Y7TvCjyqoNpb)-1<D*qM#}<!qOS
Y*6zRMt;2WAhTq#4kyf)1s!+v$0TuJ`00000

literal 0
HcmV?d00001

diff --git a/src/gcs/client.py b/src/gcs/client.py
new file mode 100644
index 0000000..2a29b9b
--- /dev/null
+++ b/src/gcs/client.py
@@ -0,0 +1,21 @@
+import os
+from os.path import exists
+from google.cloud import storage
+
+class StorageClient(storage.Client):
+    def __init__(self, bucket: str = None):
+        if not bucket:
+            bucket = os.getenv("TARGET_BUCKET")
+        
+        if not self.gcloud_key_exists():
+            raise Exception("GOOGLE_APPLICATION_CREDENTIALS has to point to a key file")
+
+        super().__init__()
+
+    # Check if env var is set to a key file
+    def gcloud_key_exists(self) -> bool:
+        keyfile = os.getenv("GOOGLE_APPLICATION_CREDENTIALS")
+
+        if not keyfile or not exists(keyfile):
+            return False
+        return True
\ No newline at end of file
diff --git a/src/glob.py b/src/glob.py
new file mode 100644
index 0000000..28d3b19
--- /dev/null
+++ b/src/glob.py
@@ -0,0 +1,4 @@
+import os.path
+
+def file_exists(file: str) -> bool:
+    return os.path.isfile(file)
\ No newline at end of file