Persistence with db

.gitignore

@@ -1,2 +1,5 @@
 indexer_local.py
-retrival_local.py
+retrival_local.py
+dropbox_trial.py
+commandline-oauth-scope.py
+*.ipynb

dropbox_handler.py

@@ -0,0 +1,148 @@
+import datetime
+import time
+import dropbox
+from dropbox.files import WriteMode
+from dropbox.exceptions import ApiError, AuthError
+import sys,os
+import firebase_handler as fbh
+
+TOKEN=fbh.fb_get("d2_accesstoken")
+APP_KEY=os.environ['DROPBOX_APP_KEY']
+APP_SECRET=os.environ['DROPBOX_APP_SECRET']
+REFRESH_TOKEN=fbh.fb_get("d2_refreshtoken")
+
+#os.environ['DROP_DIR2']="C:/dockers/chroma/chroma1/"
+#os.environ['APP_PATH']="/"
+print("token::",TOKEN)
+
+with dropbox.Dropbox(oauth2_access_token=TOKEN,app_key=APP_KEY,app_secret=APP_SECRET,oauth2_refresh_token=REFRESH_TOKEN) as dbx: #,app_key=APP_KEY,app_secret=APP_SECRET,oauth2_refresh_token=REFRESH_TOKEN) as dbx:
+    # Check that the access token is valid
+    try:
+        dbx.users_get_current_account()
+    except AuthError:
+        try:
+            dbx.check_and_refresh_access_token()
+            fbh.fb_update("d2_accesstoken",dbx._oauth2_access_token)
+        except Exception:
+            sys.exit("ERROR: Invalid access token; try re-generating an "
+                "access token from the app console on the web.")
+
+def normalizeFilename(filename):
+    while '//' in filename:
+        filename = filename.replace('//', '/')
+    return filename
+
+def getDropboxFilename(localFilename):
+    """ localFilename is $DROP_DIR2/<subpath>/<filename>"""
+    """ dropboxFilename is $APP_PATH/<subpath>/<filename"""
+    localFilename=normalizeFilename(localFilename)
+    return normalizeFilename(localFilename.replace(os.environ['DROP_DIR2'],"/",1).replace("/",os.environ['APP_PATH'],1))
+
+def getLocalFilename(dropboxFilename):
+    """ localFilename is $DROP_DIR2/<subpath>/<filename>"""
+    """ dropboxFilename is $APP_PATH/<subpath>/<filename"""
+    dropboxFilename=normalizeFilename(dropboxFilename)
+    return normalizeFilename(dropboxFilename.replace(os.environ['APP_PATH'],"/",1).replace("/",os.environ['DROP_DIR2'],1))
+
+def backupFile(localFilename):
+    """Upload a file.
+    Return the request response, or None in case of error.
+    This will also create directory on dropbox if needed
+    """
+    localFilename=normalizeFilename(localFilename) 
+    dropboxFilename=getDropboxFilename(localFilename)
+    mode = dropbox.files.WriteMode.overwrite
+    mtime = os.path.getmtime(localFilename)
+    with open(localFilename, 'rb') as f:
+        data = f.read()
+    try:
+        res = dbx.files_upload(
+            data, dropboxFilename, mode,
+            client_modified=datetime.datetime(*time.gmtime(mtime)[:6]),
+            mute=True)
+    except dropbox.exceptions.ApiError as err:
+        print('*** API error', err)
+        return None
+    print('uploaded as', res.name.encode('utf8'))
+    return res
+
+def restoreFile(dropboxFilename):
+    """Download a file.
+    Return the bytes of the file, or None if it doesn't exist.
+    Will create dir+subdirs if possible
+    """
+    dropboxFilename=normalizeFilename(dropboxFilename)
+    localFilename=getLocalFilename(dropboxFilename)
+    try:
+        md, res = dbx.files_download(dropboxFilename)
+    except dropbox.exceptions.HttpError as err:
+        print('*** HTTP error', err)
+        return None
+    data = res.content
+    print(len(data), 'bytes; md:', md)
+    localdir=os.path.dirname(localFilename)
+    if not os.path.exists(localdir):
+        os.makedirs(localdir)
+    with open(localFilename, 'wb') as f:
+        f.write(data)
+    return data
+
+def backupFolder(localFolder):
+    """ list all files in folder and subfolder and upload them"""
+    filenames=[]
+    for (root,dirs,files) in os.walk(localFolder, topdown=True):
+        print(root)
+        for filename in files:
+            filenames.append(root+"/"+filename)
+            print(root+"/"+filename)
+            backupFile(root+"/"+filename)
+
+
+def restoreFolder(dropboxFolder):
+    """ list all files in dropbox folder and subfolders and restore them"""
+    try:
+        res=dbx.files_list_folder(dropboxFolder)
+    except dropbox.exceptions.ApiError as err:
+        print('Folder listing failed for', dropboxFolder, '-- assumed empty:', err)
+    for entry in res.entries:
+        if (isinstance(entry, dropbox.files.FileMetadata)):
+            restoreFile(entry.path_display)
+        else:
+            try:
+                restoreFolder(entry.path_display)
+            except Exception:
+                print("Error restoring folder,",entry.path_display)
+        print(entry.path_display)
+        #TODO: If entry is FileMeta type then get the path_display and call restoreFile
+        #TODO: 2 If entry is FolderMeta type then call this function recursively fot it's path_display
+        #restoreFile(entry)
+
+
+# def backup(localfile,dropbox_file):
+#     with open(localfile, 'rb') as f:
+#         # We use WriteMode=overwrite to make sure that the settings in the file
+#         # are changed on upload
+#         print("Uploading " + localfile + " to Dropbox as " + dropbox_file + "...")
+#         try:
+#             dbx.files_upload(f.read(), dropbox_file, mode=WriteMode('overwrite'))
+#         except ApiError as err:
+#             # This checks for the specific error where a user doesn't have
+#             # enough Dropbox space quota to upload this file
+#             if (err.error.is_path() and
+#                     err.error.get_path().reason.is_insufficient_space()):
+#                 sys.exit("ERROR: Cannot back up; insufficient space.")
+#             elif err.user_message_text:
+#                 print(err.user_message_text)
+#                 sys.exit()
+#             else:
+#                 print(err)
+#                 sys.exit()
+
+# def restore(dropbox_file,localfile):
+    # Restore the file on Dropbox to a certain revision
+    #print("Restoring " + BACKUPPATH + " to revision " + rev + " on Dropbox...")
+    #dbx.files_restore(BACKUPPATH, rev)
+
+    # Download the specific revision of the file at BACKUPPATH to LOCALFILE
+    #print("Downloading current " + BACKUPPATH + " from Dropbox, overwriting " + LOCALFILE + "...")
+    #dbx.files_download_to_file(LOCALFILE, BACKUPPATH, rev)

firebase_handler.py

@@ -0,0 +1,21 @@
+from firebase import firebase
+import os
+
+secret=os.environ['FIREBASE_TOKEN'] 
+user_id="123"
+auth=firebase.FirebaseAuthentication(secret=secret,email="[email protected]",extra={"id":user_id})
+fb_url="https://device-1a455.firebaseio.com"
+fb= firebase.FirebaseApplication(fb_url,auth)
+base_dir="/users/"+user_id
+
+def fb_get(item):
+    """ item need to be in format d2_refreshtoken same name will be present in the firebase
+        d2 means second dropbox instance or app.
+    """
+    return fb.get(base_dir,item)
+
+def fb_update(item,value):
+    """ item need to be in format d2_refreshtoken same name will be present in the firebase
+        d2 means second dropbox instance or app.
+    """
+    return fb.patch(base_dir,{item:value})

main.py

@@ -52,6 +52,7 @@ import chromadb.errors as errors
 from uuid import UUID
 from chromadb.telemetry import Telemetry
 from overrides import override
+import dropbox_handler as dbh
 
 async def catch_exceptions_middleware(
     request: Request, call_next: Callable[[Request], Any]
@@ -73,7 +74,7 @@ def _uuid(uuid_str: str) -> UUID:
     except ValueError:
         raise InvalidUUIDError(f"Could not parse {uuid_str} as a UUID")
     
-
+dbh.restoreFolder("./index/chroma")
 app = fastapi.FastAPI(title="ChromaDB")
 app.middleware("http")(catch_exceptions_middleware)
 app.add_middleware(
@@ -102,6 +103,7 @@ def read_info(request: Request):
     request_method = request.method
     request_headers = request.headers
     request_body = request.body
+    print(request_headers)
     response= JSONResponse(content= {"nanosecond heartbeat":int(time.time_ns())})
     response.set_cookie(key="my_cookie", value="12345", httponly=True, secure=True)
     return response
@@ -109,6 +111,7 @@ def read_info(request: Request):
 @app.post(api_base+"/reset")
 def reset():
     print("Received reset request")
+    dbh.restoreFolder("./index/chroma")
     return bkend.reset()
 
 @app.get(api_base+"/version")
@@ -119,7 +122,9 @@ def version():
 @app.post(api_base+"/persist")
 def persist():
     print("Received persist request")
-    return bkend.persist()
+    retVal=bkend.persist()
+    dbh.backupFolder("./index/chroma")
+    return retVal
 
 @app.post(api_base+"/raw_sql")
 def raw_sql(raw_sql: RawSql):

requirements.txt

@@ -9,3 +9,4 @@ langchain
 sentence_transformers
 sse_starlette
 dropbox
+firebase