diff --git a/client.py b/client.py index 0b901d9..e0a890a 100755 --- a/client.py +++ b/client.py @@ -1,359 +1,82 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -import ftplib import json import os import pathlib -import socket import sys from copy import deepcopy from random import randint from shutil import rmtree from threading import Thread from time import sleep -from tkinter import Button, Entry, Label, StringVar, Tk +from datetime import datetime +import pystray from PIL import Image, ImageTk from requests import post +from websocket import WebSocket -# import rethinkdb as r from compare_json import compare_json +from config import get_config from dir_to_json import get_json - -# from tcping import Ping - -#ORIGINAL = "/home/kirbylife/Proyectos/munyal_test/another" -ORIGINAL = "/home/kirbylife/Proyectos/munyal_test/" -IP = "localhost" -USERNAME = "munyal" -PASSWORD = "123" -HOSTNAME = socket.gethostname() + str(randint(1, 100000000)) -SKIP_UPLOAD = [] -FLAG_DOWNLOAD = False -pending_routes = [] +from misc import check_network -def check_network(port): - ping = Ping(IP, port, 20) - ping.ping(1) - print(SKIP_UPLOAD) - return ping.result.rows[0].successed == 1 +class MunyalClient(pystray.Icon): + def __init__(self): + super(MunyalClient, self).__init__("Munyal") + self.icon = Image.open("img/icons/standby.png") + self.config = get_config() + self.stack = [] + def is_online(self): + ping = check_network("http://google.com", 443) + if ping: + self.icon = Image.open("img/icons/online.png") + else: + self.icon = Image.open("img/icons/offline.png") -def watch_dir(): - global pending_routes - folder = os.path.join(os.getenv("HOME"), ".munyal") - if not os.path.exists(folder): - pathlib.Path(folder).mkdir(parents=True) + def start(self): + thread_uploader = Thread(target=self.uploader, name="uploader", daemon=True) + thread_downloader = Thread(target=self.downloader, name="downloader", daemon=True) + thread_uploader.start() + thread_downloader.start() + self.run(self.__run) - actual_file = os.path.join(folder, "actual.json") - if not os.path.exists(actual_file): - with open(actual_file, "w") as f: - f.write(json.dumps([])) - pending_file = os.path.join(folder, "pending_routes.json") - if not os.path.exists(pending_file): - with open(pending_file, "w") as f: - f.write(json.dumps([])) + def __run(self, icon): + print("_run") + icon.visible = True + sleep(60) + self.stop() - with open(actual_file, "r") as f: - actual = json.loads(f.read()) - actual = get_json(ORIGINAL) - new = deepcopy(actual) - with open(pending_file, "r") as f: - pending_routes = json.loads(f.read()) - new = get_json(ORIGINAL) - while True: - sleep(0.2) + def listener(self): + pass + + def uploader(self): + print("Uploader") while True: - try: - jsons = compare_json(deepcopy(actual), deepcopy(new)) - except: - new = get_json(ORIGINAL) - else: - break - changes = get_changes(jsons) + sleep(5) + self.icon = Image.open("img/icons/online.png") - pending_routes = pending_routes + changes - with open(pending_file, "w") as f: - f.write(json.dumps(pending_routes, indent=4)) - actual = deepcopy(new) - with open(actual_file, "w") as f: - f.write(json.dumps(actual, indent=4)) + def __upload(self, path): + pass + + def downloader(self): + print("Downloader") while True: - try: - new = get_json(ORIGINAL) - except: - pass - else: - break + sleep(7) + self.icon = Image.open("img/icons/offline.png") - -def need_deleted(items, route): - out = [] - for item in items: - if item.get("is_file"): - out.append({ - "action": "delete", - "route": os.path.join(route, item.get('name')) - }) - else: - if item.get('content'): - out = out + need_deleted(item.get("content"), - os.path.join(route, item.get('name'))) - else: - out.append({ - "action": "delete_folder", - "route": os.path.join(route, item.get('name')) - }) - return out - - -def need_added(items, route): - out = [] - for item in items: - if item.get("is_file"): - out.append({ - "action": "add", - "route": os.path.join(route, item.get('name')) - }) - else: - if item.get('content'): - out = out + need_added(item.get("content"), - os.path.join(route, item.get('name'))) - else: - out.append({ - "action": "add_folder", - "route": os.path.join(route, item.get('name')) - }) - return out - - -def get_changes(jsons, route=''): - delete, add = jsons - out = need_deleted(delete, route) + need_added(add, route) - return out - - -def _is_ftp_dir(ftp_handle, name): - original_cwd = ftp_handle.pwd() - try: - ftp_handle.cwd(name) - ftp_handle.cwd(original_cwd) - return True - except: - return False - - -def _make_parent_dir(fpath): - #dirname = os.path.dirname(fpath) - dirname = os.path.join(ORIGINAL, fpath) - while not os.path.exists(dirname): - try: - os.makedirs(dirname) - print("created {0}".format(dirname)) - except: - _make_parent_dir(dirname) - - -def _download_ftp_file(ftp_handle, name, dest, overwrite): - if not os.path.exists(dest) or overwrite is True: - try: - with open(dest, 'wb') as f: - ftp_handle.retrbinary("RETR {0}".format(name), f.write) - print("downloaded: {0}".format(dest)) - except FileNotFoundError: - print("FAILED: {0}".format(dest)) - else: - print("already exists: {0}".format(dest)) - - -def _mirror_ftp_dir(ftp_handle, name, overwrite): - for item in ftp_handle.nlst(name): - SKIP_UPLOAD.append(item) - if _is_ftp_dir(ftp_handle, item): - _make_parent_dir(item.lstrip("/")) - _mirror_ftp_dir(ftp_handle, os.path.join(name, item), overwrite) - else: - _download_ftp_file(ftp_handle, item, os.path.join(ORIGINAL, item), - overwrite) - - -def download_ftp_tree(overwrite=False): - FLAG_DOWNLOAD = True - ftp_handle = ftplib.FTP(IP, USERNAME, PASSWORD) - path = "" - original_directory = os.getcwd() - os.chdir(ORIGINAL) - _mirror_ftp_dir(ftp_handle, path, overwrite) - os.chdir(original_directory) - ftp_handle.close() - FLAG_DOWNLOAD = False - - -def upload(*args): - global SKIP_UPLOAD - global pending_routes - print("Modulo de subida listo") - while True: - sleep(0.1) - if check_network('21') and pending_routes: - change = pending_routes.pop(0) - if change['route'] not in SKIP_UPLOAD: - ftp = ftplib.FTP(IP, USERNAME, PASSWORD) - route = os.path.join(ORIGINAL, change['route']) - success = False - while not success: - try: - if change['action'] == 'add': - while FLAG_DOWNLOAD: - print("Wait") - print("Agregar archivo") - with open(route, "rb") as f: - ftp.storbinary("STOR /" + change['route'], f) - elif change['action'] == 'add_folder': - print("Agregar carpeta") - ftp.mkd(change['route']) - elif change['action'] == 'delete': - print("Borrar archivo") - ftp.delete(change['route']) - elif change['action'] == 'delete_folder': - print("Borrar carpeta") - ftp.rmd(change['route']) - else: - print("Unexpected action") - except: - print("Error uploading\n") - r = post("http://" + IP + ':8000/upload', - data={ - 'host': HOSTNAME, - 'action': change['action'], - 'route': change['route'] - }) - r = json.loads(r.text) - print(json.dumps(r, indent=4)) - success = r['status'] == 'ok' - ftp.close() - else: - SKIP_UPLOAD.pop() - return 0 - - -def download(*args): - global SKIP_UPLOAD - while True: - sleep(1) - if check_network(28015) and check_network(21): - try: - download_ftp_tree(overwrite=False) - - print("Modulo de descarga listo") - print("Carpeta " + ORIGINAL) - r.connect(IP, 28015).repl() - cursor = r.table("changes").changes().run() - for document in cursor: - change = document['new_val'] - #print(change) - if change['host'] != HOSTNAME: - route = os.path.join(ORIGINAL, change['route']) - SKIP_UPLOAD.append(change['route']) - try: - if change['action'] == 'add': - print("Agregar archivo") - FLAG_DOWNLOAD = True - ftp = ftplib.FTP(IP, USERNAME, PASSWORD) - with open(route, "wb") as f: - ftp.retrbinary("RETR /" + change['route'], - f.write) - ftp.close() - FLAG_DOWNLOAD = False - elif change['action'] == 'add_folder': - print("Agregar carpeta") - pathlib.Path(route).mkdir(parents=True) - elif change['action'] == 'delete': - print("Borrar archivo") - pathlib.Path(route).unlink() - elif change['action'] == 'delete_folder': - print("Borrar carpeta") - rmtree(route) - else: - print("Unexpected action") - except OSError as e: - print("Error en el sistema operativo") - except: - print("Error en el servidor FTP") - except r.errors.ReqlDriverError as e: - print("Conection refused with rethinkdb") - return 0 - - -def run_client(window, password, username, host, folder): - global PASSWORD - global IP - global USERNAME - global ORIGINAL - - PASSWORD = password.get() - USERNAME = username.get() - IP = host.get() - ORIGINAL = folder.get() - - if not os.path.exists(ORIGINAL): - pathlib.Path(ORIGINAL).mkdir(parents=True) - - download_thread = Thread(target=download, args=[window]) - download_thread.setDaemon(True) - download_thread.start() - - upload_thread = Thread(target=upload, args=[window]) - upload_thread.setDaemon(True) - upload_thread.start() - - watch_dir_thread = Thread(target=watch_dir) - watch_dir_thread.setDaemon(True) - watch_dir_thread.start() - - -def main(args): - root = Tk() - root.geometry("250x400") - - img_logo = Image.open("img/logo.png") - img_tk = ImageTk.PhotoImage(img_logo.resize((200, 150), Image.ANTIALIAS)) - - host = StringVar() - host.set("localhost") - host_field = Entry(root, textvariable=host) - - passwd = StringVar() - passwd_field = Entry(root, textvariable=passwd, show="*") - - folder = StringVar() - folder.set(os.path.join(os.getenv("HOME"), "Munyal")) - folder_field = Entry(root, textvariable=folder) - - connect = Button(root, - text="Conectar", - command=lambda: run_client(root, passwd, host, folder)) - - Label(root, text="MUNYAL").pack() - Label(root, image=img_tk).pack() - Label(root, text="").pack() - Label(root, text="Ruta del servidor").pack() - host_field.pack() - Label(root, text="").pack() - Label(root, text="Contraseña").pack() - passwd_field.pack() - Label(root, text="").pack() - Label(root, text="Carpeta a sincronizar").pack() - folder_field.pack() - Label(root, text="").pack() - connect.pack() - - root.mainloop() + def __download(self, path): + pass if __name__ == '__main__': - import sys - sys.exit(main(sys.argv)) + try: + client = MunyalClient() + client.start() + sys.exit(0) + except BaseException: + sys.exit(1) diff --git a/http_server.py b/http_server.py index fa9ce25..d0c94d8 100644 --- a/http_server.py +++ b/http_server.py @@ -1,19 +1,18 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -from flask import Flask -from flask import request -from flask import jsonify - -import rethinkdb as r - +import json +from hashlib import md5 from random import randint from time import time -import json +from flask import Flask, jsonify, request + +import rethinkdb as r app = Flask(__name__) + def md5sum(filename): try: hash = md5() @@ -24,9 +23,10 @@ def md5sum(filename): except: return None + @app.route("/", methods=["GET"]) def index(): - return(''' + return (''' Munyal API @@ -37,17 +37,18 @@ def index(): ''') + @app.route("/upload", methods=["POST"]) def upload(): try: - r.connect( "localhost", 28015).repl() + r.connect("localhost", 28015).repl() cursor = r.table("changes") - + host = request.form.get("host") action = request.form.get("action") route = request.form.get("route") obj = { - 'id' : str(time()).split('.')[0] + str(randint(1, 1000000)), + 'id': str(time()).split('.')[0] + str(randint(1, 1000000)), 'action': action, 'route': route, 'host': host @@ -61,7 +62,7 @@ def upload(): status = 'error' obj['status'] = status return jsonify(obj) - + if __name__ == '__main__': app.run(debug=True) diff --git a/img/icons/offline.png b/img/icons/offline.png new file mode 100644 index 0000000..3f0e329 Binary files /dev/null and b/img/icons/offline.png differ diff --git a/img/icons/offline_f.png b/img/icons/offline_f.png new file mode 100644 index 0000000..72fdddf Binary files /dev/null and b/img/icons/offline_f.png differ diff --git a/img/icons/online.png b/img/icons/online.png new file mode 100644 index 0000000..f6724c3 Binary files /dev/null and b/img/icons/online.png differ diff --git a/img/icons/online_f.png b/img/icons/online_f.png new file mode 100644 index 0000000..a66f3f7 Binary files /dev/null and b/img/icons/online_f.png differ diff --git a/img/icons/standby.png b/img/icons/standby.png new file mode 100644 index 0000000..c099117 Binary files /dev/null and b/img/icons/standby.png differ diff --git a/img/icons/standby_f.png b/img/icons/standby_f.png new file mode 100644 index 0000000..487fda7 Binary files /dev/null and b/img/icons/standby_f.png differ diff --git a/main.py b/main.py new file mode 100644 index 0000000..414643c --- /dev/null +++ b/main.py @@ -0,0 +1,13 @@ +import os +import sys + +from client import MunyalClient +from misc import get_os + + +def main(): + os = get_os() + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/misc.py b/misc.py index 7b1bfda..5f5c65d 100644 --- a/misc.py +++ b/misc.py @@ -8,7 +8,6 @@ from tcping import Ping def check_network(ip, port): - return True ping = Ping(ip, port, 20) ping.ping(1) return ping.result.rows[0].successed == 1 diff --git a/temp/client.py b/temp/client.py new file mode 100755 index 0000000..ae25afd --- /dev/null +++ b/temp/client.py @@ -0,0 +1,363 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +import ftplib +import json +import os +import pathlib +import socket +import sys +from copy import deepcopy +from random import randint +from shutil import rmtree +from threading import Thread +from time import sleep +from tkinter import Button, Entry, Label, StringVar, Tk + +from requests import post + +import rethinkdb as r +from compare_json import compare_json +from dir_to_json import get_json +from tcping import Ping + +#ORIGINAL = "/home/kirbylife/Proyectos/munyal_test/another" +ORIGINAL = "/home/kirbylife/Proyectos/munyal_test/" +IP = "localhost" +USERNAME = "munyal" +PASSWORD = "123" +HOSTNAME = socket.gethostname() + str(randint(1, 100000000)) +SKIP_UPLOAD = [] +FLAG_DOWNLOAD = False +pending_routes = [] + + +def check_network(port): + ping = Ping(IP, port, 20) + ping.ping(1) + print(SKIP_UPLOAD) + return ping.result.rows[0].successed == 1 + + +def watch_dir(): + global pending_routes + folder = os.path.join(os.getenv("HOME"), ".munyal") + if not os.path.exists(folder): + pathlib.Path(folder).mkdir(parents=True) + + actual_file = os.path.join(folder, "actual.json") + if not os.path.exists(actual_file): + with open(actual_file, "w") as f: + f.write(json.dumps([])) + pending_file = os.path.join(folder, "pending_routes.json") + if not os.path.exists(pending_file): + with open(pending_file, "w") as f: + f.write(json.dumps([])) + + with open(actual_file, "r") as f: + actual = json.loads(f.read()) + actual = get_json(ORIGINAL) + new = deepcopy(actual) + with open(pending_file, "r") as f: + pending_routes = json.loads(f.read()) + new = get_json(ORIGINAL) + while True: + sleep(0.2) + while True: + try: + jsons = compare_json(deepcopy(actual), deepcopy(new)) + except: + new = get_json(ORIGINAL) + else: + break + changes = get_changes(jsons) + + pending_routes = pending_routes + changes + with open(pending_file, "w") as f: + f.write(json.dumps(pending_routes, indent=4)) + + actual = deepcopy(new) + with open(actual_file, "w") as f: + f.write(json.dumps(actual, indent=4)) + while True: + try: + new = get_json(ORIGINAL) + except: + pass + else: + break + + +def need_deleted(items, route): + out = [] + for item in items: + if item.get("is_file"): + out.append({ + "action": "delete", + "route": os.path.join(route, item.get('name')) + }) + else: + if item.get('content'): + out = out + need_deleted(item.get("content"), + os.path.join(route, item.get('name'))) + else: + out.append({ + "action": "delete_folder", + "route": os.path.join(route, item.get('name')) + }) + return out + + +def need_added(items, route): + out = [] + for item in items: + if item.get("is_file"): + out.append({ + "action": "add", + "route": os.path.join(route, item.get('name')) + }) + else: + if item.get('content'): + out = out + need_added(item.get("content"), + os.path.join(route, item.get('name'))) + else: + out.append({ + "action": "add_folder", + "route": os.path.join(route, item.get('name')) + }) + return out + + +def get_changes(jsons, route=''): + delete, add = jsons + out = need_deleted(delete, route) + need_added(add, route) + return out + + +def _is_ftp_dir(ftp_handle, name): + original_cwd = ftp_handle.pwd() + try: + ftp_handle.cwd(name) + ftp_handle.cwd(original_cwd) + return True + except: + return False + + +def _make_parent_dir(fpath): + #dirname = os.path.dirname(fpath) + dirname = os.path.join(ORIGINAL, fpath) + while not os.path.exists(dirname): + try: + os.makedirs(dirname) + print("created {0}".format(dirname)) + except: + _make_parent_dir(dirname) + + +def _download_ftp_file(ftp_handle, name, dest, overwrite): + if not os.path.exists(dest) or overwrite is True: + try: + with open(dest, 'wb') as f: + ftp_handle.retrbinary("RETR {0}".format(name), f.write) + print("downloaded: {0}".format(dest)) + except FileNotFoundError: + print("FAILED: {0}".format(dest)) + else: + print("already exists: {0}".format(dest)) + + +def _mirror_ftp_dir(ftp_handle, name, overwrite): + for item in ftp_handle.nlst(name): + SKIP_UPLOAD.append(item) + if _is_ftp_dir(ftp_handle, item): + _make_parent_dir(item.lstrip("/")) + _mirror_ftp_dir(ftp_handle, os.path.join(name, item), overwrite) + else: + _download_ftp_file(ftp_handle, item, os.path.join(ORIGINAL, item), + overwrite) + + +def download_ftp_tree(overwrite=False): + FLAG_DOWNLOAD = True + ftp_handle = ftplib.FTP(IP, USERNAME, PASSWORD) + path = "" + original_directory = os.getcwd() + os.chdir(ORIGINAL) + _mirror_ftp_dir(ftp_handle, path, overwrite) + os.chdir(original_directory) + ftp_handle.close() + FLAG_DOWNLOAD = False + + +def upload(*args): + global SKIP_UPLOAD + global pending_routes + print("Modulo de subida listo") + while True: + sleep(0.1) + if check_network('21') and pending_routes: + change = pending_routes.pop(0) + while FLAG_DOWNLOAD: + print("Wait") + if change['route'] not in SKIP_UPLOAD: + ftp = ftplib.FTP(IP, USERNAME, PASSWORD) + route = os.path.join(ORIGINAL, change['route']) + success = False + while not success: + try: + if change['action'] == 'add': + print("Agregar archivo") + with open(route, "rb") as f: + ftp.storbinary("STOR /" + change['route'], f) + elif change['action'] == 'add_folder': + print("Agregar carpeta") + ftp.mkd(change['route']) + elif change['action'] == 'delete': + print("Borrar archivo") + ftp.delete(change['route']) + elif change['action'] == 'delete_folder': + print("Borrar carpeta") + ftp.rmd(change['route']) + else: + print("Unexpected action") + except: + print("Error uploading\n") + r = post("http://" + IP + ':5000/upload', + data={ + 'host': HOSTNAME, + 'action': change['action'], + 'route': change['route'] + }) + r = json.loads(r.text) + print(json.dumps(r, indent=4)) + success = r['status'] == 'ok' + ftp.close() + else: + SKIP_UPLOAD.pop() + return 0 + + +def download(*args): + global SKIP_UPLOAD + while True: + sleep(1) + if check_network(28015) and check_network(21): + try: + download_ftp_tree(overwrite=False) + + print("Modulo de descarga listo") + print("Carpeta " + ORIGINAL) + r.connect(IP, 28015).repl() + cursor = r.table("changes").changes().run() + for document in cursor: + change = document['new_val'] + #print(change) + if change['host'] != HOSTNAME: + FLAG_DOWNLOAD = True + route = os.path.join(ORIGINAL, change['route']) + SKIP_UPLOAD.append(change['route']) + try: + if change['action'] == 'add': + print("Agregar archivo") + ftp = ftplib.FTP(IP, USERNAME, PASSWORD) + with open(route, "wb") as f: + ftp.retrbinary("RETR /" + change['route'], + f.write) + ftp.close() + elif change['action'] == 'add_folder': + print("Agregar carpeta") + pathlib.Path(route).mkdir(parents=True) + elif change['action'] == 'delete': + print("Borrar archivo") + pathlib.Path(route).unlink() + elif change['action'] == 'delete_folder': + print("Borrar carpeta") + rmtree(route) + else: + print("Unexpected action") + except OSError as e: + print("Error en el sistema operativo") + except: + print("Error en el servidor FTP") + FLAG_DOWNLOAD = False + except rethinkdb.errors.ReqlDriverError as e: + print("Conection refused with rethinkdb") + return 0 + + +def run_client(window, password, username, host, folder): + global PASSWORD + global IP + global USERNAME + global ORIGINAL + + PASSWORD = password.get() + USERNAME = username.get() + IP = host.get() + ORIGINAL = folder.get() + + if not os.path.exists(ORIGINAL): + pathlib.Path(ORIGINAL).mkdir(parents=True) + + download_thread = Thread(target=download, args=[window]) + download_thread.setDaemon(True) + download_thread.start() + + upload_thread = Thread(target=upload, args=[window]) + upload_thread.setDaemon(True) + upload_thread.start() + + watch_dir_thread = Thread(target=watch_dir) + watch_dir_thread.setDaemon(True) + watch_dir_thread.start() + + +def main(args): + root = Tk() + root.geometry("200x300") + root.title("MUNYAL") + + host = StringVar() + host.set("localhost") + host_field = Entry(root, textvariable=host) + + user = StringVar() + user.set("munyal") + user_field = Entry(root, textvariable=user) + + passwd = StringVar() + passwd.set("123") + passwd_field = Entry(root, textvariable=passwd, show="*") + + folder = StringVar() + folder.set(os.path.join(os.getenv("HOME"), "Munyal")) + folder_field = Entry(root, textvariable=folder) + + connect = Button( + root, + text="Conectar", + command=lambda: run_client(root, passwd, user, host, folder)) + + Label(root, text="MUNYAL").pack() + Label(root, text="").pack() + Label(root, text="Ruta del servidor").pack() + host_field.pack() + Label(root, text="").pack() + Label(root, text="Nombre de usuario").pack() + user_field.pack() + Label(root, text="").pack() + Label(root, text="Contraseña").pack() + passwd_field.pack() + Label(root, text="").pack() + Label(root, text="Carpeta a sincronizar").pack() + folder_field.pack() + Label(root, text="").pack() + connect.pack() + + root.mainloop() + + +if __name__ == '__main__': + import sys + sys.exit(main(sys.argv)) diff --git a/temp/compare_json.py b/temp/compare_json.py new file mode 100644 index 0000000..77c1d15 --- /dev/null +++ b/temp/compare_json.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +import json + +from dir_to_json import get_json + +def compare_json(json1, json2): + bckup1, bckup2 = json1[:], json2[:] + items1 = list(enumerate(json1)) + items2 = list(enumerate(json2)) + for i, item1 in items1: + for j, item2 in items2: + if item1["name"] == item2["name"]: + if item1["is_file"] == True == item2["is_file"]: + if item1["checksum"] == item2["checksum"]: + json1[i] = None + json2[j] = None + ''' + else: + json1[i]["tag"] = "update" + json2[j] = None + ''' + elif item1["is_file"] == False == item2["is_file"]: + new_json1, new_json2 = compare_json(item1["content"], item2["content"]) + if len(new_json1) == 0: + json1[i] = None + else: + json1[i]["content"] = new_json1 + if len(new_json2) == 0: + json2[j] = None + else: + json2[j]["content"] = new_json2 + elif item1["is_file"] != item2["is_file"]:##### Caso hipotetico imposible ##### + json1[i]["tag"] == "delete" + json1 = list(filter(None, json1)) + json2 = list(filter(None, json2)) + return json1, json2 +if __name__ == "__main__": + try: + json1 = get_json("/home/kirbylife/Proyectos/munyal_test/original") + json2 = get_json("/home/kirbylife/Proyectos/munyal_test/copy") + except: + print("error outside") + json1, json2 = compare_json(json1, json2) + #print(len(json1), len(json2)) + print(json.dumps(json1, indent=4)) + print("\n============\n") + print(json.dumps(json2, indent=4)) diff --git a/temp/dir_to_json.py b/temp/dir_to_json.py new file mode 100644 index 0000000..5f59b1a --- /dev/null +++ b/temp/dir_to_json.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +from hashlib import md5 + +import os +import json + + +def md5sum(filename): + try: + hash = md5() + with open(filename, "rb") as f: + for chunk in iter(lambda: f.read(128 * hash.block_size), b""): + hash.update(chunk) + return hash.hexdigest() + except: + return None + +def get_json(path): + out = [] + items = os.listdir(path) + try: + for item in items: + if item[0] != ".": + item_json = { + "name": item + } + route = os.path.join(path, item) + if os.path.isdir(route): + item_json["is_file"] = False + item_json["content"] = get_json(route) + elif os.path.isfile(route): + item_json["is_file"] = True + item_json["size"] = os.path.getsize(route) + item_json["last_modified"] = os.path.getmtime(route) + item_json["created_at"] = os.path.getctime(route) + checksum = md5sum(route) + if checksum: + item_json["checksum"] = checksum + else: + item = None + out.append(item_json) + except: + return get_json(path) + return out + +if __name__ == "__main__": + output = get_json("/media/kirbylife/DATOS/Proyectos/PyCharmProjects/Munyal/folder_test") + print(json.dumps(output, indent=4)) diff --git a/temp/download_ftp_tree.py b/temp/download_ftp_tree.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/temp/download_ftp_tree.py @@ -0,0 +1 @@ + diff --git a/temp/first_test.py b/temp/first_test.py new file mode 100644 index 0000000..38d97ba --- /dev/null +++ b/temp/first_test.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +from compare_json import compare_json +from dir_to_json import get_json + +from copy import deepcopy +from time import sleep + +ORIGINAL = "/home/kirbylife/Proyectos/munyal_test/original" + +def main(args): + actual = get_json(ORIGINAL) + new = deepcopy(actual) + + while True: + delete, add = compare_json(deepcopy(actual), deepcopy(new)) + for item in delete: + if item.get("tag"): + if item.get("tag") == "update": + print("Actualizado el archivo {}".format(item.get("name"))) + else: + print("borrado el archivo {}".format(item.get("name"))) + + for item in add: + print("Agregado el archivo {}".format(item.get("name"))) + actual = deepcopy(new) + new = get_json(ORIGINAL) + return 0 + +if __name__ == '__main__': + import sys + sys.exit(main(sys.argv)) + + + + + + + + + diff --git a/temp/http_server.py b/temp/http_server.py new file mode 100644 index 0000000..fa9ce25 --- /dev/null +++ b/temp/http_server.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +from flask import Flask +from flask import request +from flask import jsonify + +import rethinkdb as r + +from random import randint +from time import time + +import json + +app = Flask(__name__) + +def md5sum(filename): + try: + hash = md5() + with open(filename, "rb") as f: + for chunk in iter(lambda: f.read(128 * hash.block_size), b""): + hash.update(chunk) + return hash.hexdigest() + except: + return None + +@app.route("/", methods=["GET"]) +def index(): + return(''' + + + Munyal API + + +

Munyal private API

+ + + ''') + +@app.route("/upload", methods=["POST"]) +def upload(): + try: + r.connect( "localhost", 28015).repl() + cursor = r.table("changes") + + host = request.form.get("host") + action = request.form.get("action") + route = request.form.get("route") + obj = { + 'id' : str(time()).split('.')[0] + str(randint(1, 1000000)), + 'action': action, + 'route': route, + 'host': host + } + status = 'ok' + try: + cursor.insert(obj).run() + except: + status = 'error' + except: + status = 'error' + obj['status'] = status + return jsonify(obj) + + +if __name__ == '__main__': + app.run(debug=True) diff --git a/temp/second_test.py b/temp/second_test.py new file mode 100644 index 0000000..869acdd --- /dev/null +++ b/temp/second_test.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +from compare_json import compare_json +from dir_to_json import get_json + +from copy import deepcopy +from time import sleep +from requests import post + +import os +import socket +import json +import ftplib + +def need_deleted(items, route): + out = [] + for item in items: + if item.get("is_file"): + out.append({"action": "delete", "route": os.path.join(route, item.get('name'))}) + else: + if item.get('content'): + out = out + need_deleted(item.get("content"), os.path.join(route, item.get('name'))) + else: + out.append({"action": "delete_folder", "route": os.path.join(route, item.get('name'))}) + return out + +def need_added(items, route): + out = [] + for item in items: + if item.get("is_file"): + out.append({"action": "add", "route": os.path.join(route, item.get('name'))}) + else: + if item.get('content'): + out = out + need_added(item.get("content"), os.path.join(route, item.get('name'))) + else: + out.append({"action": "add_folder", "route": os.path.join(route, item.get('name'))}) + return out + +def get_changes(jsons, route=''): + delete, add = jsons + out = need_deleted(delete, route) + need_added(add, route) + return out + ''' + out = [] + + delete, add = jsons + for item in delete: + if item.get("is_file"): + + if item.get("tag"): + if item.get("tag") == "update": + out.append({"action": "update", "file": os.path.join(route, item.get('name'))}) + elif item.get("tag") == "delete":##### Caso hipotetico imposible ##### + if item.get("is_file"): + out.append({"action": "delete", "file": os.path.join(route, item.get('name'))}) + else: + out.append({"action": "delete_folder", "file": os.path.join(route, item.get('name'))}) + + else: + out.append({"action": "delete", "file": os.path.join(route, item.get('name'))}) + + out.append({"action": "delete", "file": os.path.join(route, item.get('name'))}) + else: + return out + ''' + +ORIGINAL = "/home/kirbylife/Proyectos/munyal_test/original" + +def main(args): + ftp = ftplib.FTP('localhost', 'munyal', '123') + actual = get_json(ORIGINAL) + new = deepcopy(actual) + switch = lambda x,o,d=None: o.get(x) if o.get(x) else d if d else lambda *args: None + while True: + sleep(1) + jsons = compare_json(deepcopy(actual), deepcopy(new)) + changes = get_changes(jsons) + for change in changes: + route = os.path.join(ORIGINAL, change['route']) + success = False + while not success: + # ~ try: + x = change['route'] + if change['action'] == 'add': + print("Agregar archivo") + with open(route, "rb") as f: + ftp.storbinary("STOR /" + x, f) + elif change['action'] == 'add_folder': + print("Agregar carpeta") + ftp.mkd(x) + elif change['action'] == 'delete': + print("Borrar archivo") + ftp.delete(x) + elif change['action'] == 'delete_folder': + print("Borrar carpeta") + ftp.rmd(x) + else: + print("Unexpected action") + r = post('http://127.0.0.1:5000/upload', data={ + 'host': socket.gethostname(), + 'action': change['action'], + 'route': change['route'] + } + ) + r = json.loads(r.text) + print(json.dumps(r, indent=4)) + success = r['status'] == 'ok' + # ~ except: + # ~ print("Error uploading, retrying again\n") + actual = deepcopy(new) + new = get_json(ORIGINAL) + return 0 + +if __name__ == '__main__': + import sys + sys.exit(main(sys.argv))