diff --git a/client.py b/client.py new file mode 100755 index 0000000..0b901d9 --- /dev/null +++ b/client.py @@ -0,0 +1,359 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +import ftplib +import json +import os +import pathlib +import socket +import sys +from copy import deepcopy +from random import randint +from shutil import rmtree +from threading import Thread +from time import sleep +from tkinter import Button, Entry, Label, StringVar, Tk + +from PIL import Image, ImageTk +from requests import post + +# import rethinkdb as r +from compare_json import compare_json +from dir_to_json import get_json + +# from tcping import Ping + +#ORIGINAL = "/home/kirbylife/Proyectos/munyal_test/another" +ORIGINAL = "/home/kirbylife/Proyectos/munyal_test/" +IP = "localhost" +USERNAME = "munyal" +PASSWORD = "123" +HOSTNAME = socket.gethostname() + str(randint(1, 100000000)) +SKIP_UPLOAD = [] +FLAG_DOWNLOAD = False +pending_routes = [] + + +def check_network(port): + ping = Ping(IP, port, 20) + ping.ping(1) + print(SKIP_UPLOAD) + return ping.result.rows[0].successed == 1 + + +def watch_dir(): + global pending_routes + folder = os.path.join(os.getenv("HOME"), ".munyal") + if not os.path.exists(folder): + pathlib.Path(folder).mkdir(parents=True) + + actual_file = os.path.join(folder, "actual.json") + if not os.path.exists(actual_file): + with open(actual_file, "w") as f: + f.write(json.dumps([])) + pending_file = os.path.join(folder, "pending_routes.json") + if not os.path.exists(pending_file): + with open(pending_file, "w") as f: + f.write(json.dumps([])) + + with open(actual_file, "r") as f: + actual = json.loads(f.read()) + actual = get_json(ORIGINAL) + new = deepcopy(actual) + with open(pending_file, "r") as f: + pending_routes = json.loads(f.read()) + new = get_json(ORIGINAL) + while True: + sleep(0.2) + while True: + try: + jsons = compare_json(deepcopy(actual), deepcopy(new)) + except: + new = get_json(ORIGINAL) + else: + break + changes = get_changes(jsons) + + pending_routes = pending_routes + changes + with open(pending_file, "w") as f: + f.write(json.dumps(pending_routes, indent=4)) + + actual = deepcopy(new) + with open(actual_file, "w") as f: + f.write(json.dumps(actual, indent=4)) + while True: + try: + new = get_json(ORIGINAL) + except: + pass + else: + break + + +def need_deleted(items, route): + out = [] + for item in items: + if item.get("is_file"): + out.append({ + "action": "delete", + "route": os.path.join(route, item.get('name')) + }) + else: + if item.get('content'): + out = out + need_deleted(item.get("content"), + os.path.join(route, item.get('name'))) + else: + out.append({ + "action": "delete_folder", + "route": os.path.join(route, item.get('name')) + }) + return out + + +def need_added(items, route): + out = [] + for item in items: + if item.get("is_file"): + out.append({ + "action": "add", + "route": os.path.join(route, item.get('name')) + }) + else: + if item.get('content'): + out = out + need_added(item.get("content"), + os.path.join(route, item.get('name'))) + else: + out.append({ + "action": "add_folder", + "route": os.path.join(route, item.get('name')) + }) + return out + + +def get_changes(jsons, route=''): + delete, add = jsons + out = need_deleted(delete, route) + need_added(add, route) + return out + + +def _is_ftp_dir(ftp_handle, name): + original_cwd = ftp_handle.pwd() + try: + ftp_handle.cwd(name) + ftp_handle.cwd(original_cwd) + return True + except: + return False + + +def _make_parent_dir(fpath): + #dirname = os.path.dirname(fpath) + dirname = os.path.join(ORIGINAL, fpath) + while not os.path.exists(dirname): + try: + os.makedirs(dirname) + print("created {0}".format(dirname)) + except: + _make_parent_dir(dirname) + + +def _download_ftp_file(ftp_handle, name, dest, overwrite): + if not os.path.exists(dest) or overwrite is True: + try: + with open(dest, 'wb') as f: + ftp_handle.retrbinary("RETR {0}".format(name), f.write) + print("downloaded: {0}".format(dest)) + except FileNotFoundError: + print("FAILED: {0}".format(dest)) + else: + print("already exists: {0}".format(dest)) + + +def _mirror_ftp_dir(ftp_handle, name, overwrite): + for item in ftp_handle.nlst(name): + SKIP_UPLOAD.append(item) + if _is_ftp_dir(ftp_handle, item): + _make_parent_dir(item.lstrip("/")) + _mirror_ftp_dir(ftp_handle, os.path.join(name, item), overwrite) + else: + _download_ftp_file(ftp_handle, item, os.path.join(ORIGINAL, item), + overwrite) + + +def download_ftp_tree(overwrite=False): + FLAG_DOWNLOAD = True + ftp_handle = ftplib.FTP(IP, USERNAME, PASSWORD) + path = "" + original_directory = os.getcwd() + os.chdir(ORIGINAL) + _mirror_ftp_dir(ftp_handle, path, overwrite) + os.chdir(original_directory) + ftp_handle.close() + FLAG_DOWNLOAD = False + + +def upload(*args): + global SKIP_UPLOAD + global pending_routes + print("Modulo de subida listo") + while True: + sleep(0.1) + if check_network('21') and pending_routes: + change = pending_routes.pop(0) + if change['route'] not in SKIP_UPLOAD: + ftp = ftplib.FTP(IP, USERNAME, PASSWORD) + route = os.path.join(ORIGINAL, change['route']) + success = False + while not success: + try: + if change['action'] == 'add': + while FLAG_DOWNLOAD: + print("Wait") + print("Agregar archivo") + with open(route, "rb") as f: + ftp.storbinary("STOR /" + change['route'], f) + elif change['action'] == 'add_folder': + print("Agregar carpeta") + ftp.mkd(change['route']) + elif change['action'] == 'delete': + print("Borrar archivo") + ftp.delete(change['route']) + elif change['action'] == 'delete_folder': + print("Borrar carpeta") + ftp.rmd(change['route']) + else: + print("Unexpected action") + except: + print("Error uploading\n") + r = post("http://" + IP + ':8000/upload', + data={ + 'host': HOSTNAME, + 'action': change['action'], + 'route': change['route'] + }) + r = json.loads(r.text) + print(json.dumps(r, indent=4)) + success = r['status'] == 'ok' + ftp.close() + else: + SKIP_UPLOAD.pop() + return 0 + + +def download(*args): + global SKIP_UPLOAD + while True: + sleep(1) + if check_network(28015) and check_network(21): + try: + download_ftp_tree(overwrite=False) + + print("Modulo de descarga listo") + print("Carpeta " + ORIGINAL) + r.connect(IP, 28015).repl() + cursor = r.table("changes").changes().run() + for document in cursor: + change = document['new_val'] + #print(change) + if change['host'] != HOSTNAME: + route = os.path.join(ORIGINAL, change['route']) + SKIP_UPLOAD.append(change['route']) + try: + if change['action'] == 'add': + print("Agregar archivo") + FLAG_DOWNLOAD = True + ftp = ftplib.FTP(IP, USERNAME, PASSWORD) + with open(route, "wb") as f: + ftp.retrbinary("RETR /" + change['route'], + f.write) + ftp.close() + FLAG_DOWNLOAD = False + elif change['action'] == 'add_folder': + print("Agregar carpeta") + pathlib.Path(route).mkdir(parents=True) + elif change['action'] == 'delete': + print("Borrar archivo") + pathlib.Path(route).unlink() + elif change['action'] == 'delete_folder': + print("Borrar carpeta") + rmtree(route) + else: + print("Unexpected action") + except OSError as e: + print("Error en el sistema operativo") + except: + print("Error en el servidor FTP") + except r.errors.ReqlDriverError as e: + print("Conection refused with rethinkdb") + return 0 + + +def run_client(window, password, username, host, folder): + global PASSWORD + global IP + global USERNAME + global ORIGINAL + + PASSWORD = password.get() + USERNAME = username.get() + IP = host.get() + ORIGINAL = folder.get() + + if not os.path.exists(ORIGINAL): + pathlib.Path(ORIGINAL).mkdir(parents=True) + + download_thread = Thread(target=download, args=[window]) + download_thread.setDaemon(True) + download_thread.start() + + upload_thread = Thread(target=upload, args=[window]) + upload_thread.setDaemon(True) + upload_thread.start() + + watch_dir_thread = Thread(target=watch_dir) + watch_dir_thread.setDaemon(True) + watch_dir_thread.start() + + +def main(args): + root = Tk() + root.geometry("250x400") + + img_logo = Image.open("img/logo.png") + img_tk = ImageTk.PhotoImage(img_logo.resize((200, 150), Image.ANTIALIAS)) + + host = StringVar() + host.set("localhost") + host_field = Entry(root, textvariable=host) + + passwd = StringVar() + passwd_field = Entry(root, textvariable=passwd, show="*") + + folder = StringVar() + folder.set(os.path.join(os.getenv("HOME"), "Munyal")) + folder_field = Entry(root, textvariable=folder) + + connect = Button(root, + text="Conectar", + command=lambda: run_client(root, passwd, host, folder)) + + Label(root, text="MUNYAL").pack() + Label(root, image=img_tk).pack() + Label(root, text="").pack() + Label(root, text="Ruta del servidor").pack() + host_field.pack() + Label(root, text="").pack() + Label(root, text="ContraseƱa").pack() + passwd_field.pack() + Label(root, text="").pack() + Label(root, text="Carpeta a sincronizar").pack() + folder_field.pack() + Label(root, text="").pack() + connect.pack() + + root.mainloop() + + +if __name__ == '__main__': + import sys + sys.exit(main(sys.argv)) diff --git a/compare_json.py b/compare_json.py new file mode 100644 index 0000000..77c1d15 --- /dev/null +++ b/compare_json.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +import json + +from dir_to_json import get_json + +def compare_json(json1, json2): + bckup1, bckup2 = json1[:], json2[:] + items1 = list(enumerate(json1)) + items2 = list(enumerate(json2)) + for i, item1 in items1: + for j, item2 in items2: + if item1["name"] == item2["name"]: + if item1["is_file"] == True == item2["is_file"]: + if item1["checksum"] == item2["checksum"]: + json1[i] = None + json2[j] = None + ''' + else: + json1[i]["tag"] = "update" + json2[j] = None + ''' + elif item1["is_file"] == False == item2["is_file"]: + new_json1, new_json2 = compare_json(item1["content"], item2["content"]) + if len(new_json1) == 0: + json1[i] = None + else: + json1[i]["content"] = new_json1 + if len(new_json2) == 0: + json2[j] = None + else: + json2[j]["content"] = new_json2 + elif item1["is_file"] != item2["is_file"]:##### Caso hipotetico imposible ##### + json1[i]["tag"] == "delete" + json1 = list(filter(None, json1)) + json2 = list(filter(None, json2)) + return json1, json2 +if __name__ == "__main__": + try: + json1 = get_json("/home/kirbylife/Proyectos/munyal_test/original") + json2 = get_json("/home/kirbylife/Proyectos/munyal_test/copy") + except: + print("error outside") + json1, json2 = compare_json(json1, json2) + #print(len(json1), len(json2)) + print(json.dumps(json1, indent=4)) + print("\n============\n") + print(json.dumps(json2, indent=4)) diff --git a/config.py b/config.py new file mode 100644 index 0000000..6b7a7ec --- /dev/null +++ b/config.py @@ -0,0 +1,43 @@ +import os +from tkinter import Button, Entry, Label, StringVar, PhotoImage, Tk, Canvas + + +def main(args): + root = Tk() + root.title("Munyal") + root.geometry("320x500") + + logo = PhotoImage(file="img/logo.png") + + host = StringVar() + host_field = Entry(root, textvariable=host, width=30) + + passwd = StringVar() + passwd_field = Entry(root, textvariable=passwd, show="*", width=30) + + folder = StringVar() + folder.set(os.path.join(os.getenv("HOME"), "Munyal")) + folder_field = Entry(root, textvariable=folder, width=30) + + connect = Button(root, text="Conectar", command=lambda: None, width=10) + + Label(root, image=logo).pack() + Label(root, text="MUNYAL", font=("", 30)).pack() + Label(root, text="").pack() + Label(root, text="Nombre del servidor").pack() + host_field.pack() + Label(root, text="").pack() + Label(root, text="ContraseƱa").pack() + passwd_field.pack() + Label(root, text="").pack() + Label(root, text="Carpeta a sincronizar").pack() + folder_field.pack() + Label(root, text="").pack() + connect.pack() + + root.mainloop() + + +if __name__ == '__main__': + import sys + sys.exit(main(sys.argv)) diff --git a/dir_to_json.py b/dir_to_json.py new file mode 100644 index 0000000..5f59b1a --- /dev/null +++ b/dir_to_json.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +from hashlib import md5 + +import os +import json + + +def md5sum(filename): + try: + hash = md5() + with open(filename, "rb") as f: + for chunk in iter(lambda: f.read(128 * hash.block_size), b""): + hash.update(chunk) + return hash.hexdigest() + except: + return None + +def get_json(path): + out = [] + items = os.listdir(path) + try: + for item in items: + if item[0] != ".": + item_json = { + "name": item + } + route = os.path.join(path, item) + if os.path.isdir(route): + item_json["is_file"] = False + item_json["content"] = get_json(route) + elif os.path.isfile(route): + item_json["is_file"] = True + item_json["size"] = os.path.getsize(route) + item_json["last_modified"] = os.path.getmtime(route) + item_json["created_at"] = os.path.getctime(route) + checksum = md5sum(route) + if checksum: + item_json["checksum"] = checksum + else: + item = None + out.append(item_json) + except: + return get_json(path) + return out + +if __name__ == "__main__": + output = get_json("/media/kirbylife/DATOS/Proyectos/PyCharmProjects/Munyal/folder_test") + print(json.dumps(output, indent=4)) diff --git a/first_test.py b/first_test.py new file mode 100644 index 0000000..38d97ba --- /dev/null +++ b/first_test.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +from compare_json import compare_json +from dir_to_json import get_json + +from copy import deepcopy +from time import sleep + +ORIGINAL = "/home/kirbylife/Proyectos/munyal_test/original" + +def main(args): + actual = get_json(ORIGINAL) + new = deepcopy(actual) + + while True: + delete, add = compare_json(deepcopy(actual), deepcopy(new)) + for item in delete: + if item.get("tag"): + if item.get("tag") == "update": + print("Actualizado el archivo {}".format(item.get("name"))) + else: + print("borrado el archivo {}".format(item.get("name"))) + + for item in add: + print("Agregado el archivo {}".format(item.get("name"))) + actual = deepcopy(new) + new = get_json(ORIGINAL) + return 0 + +if __name__ == '__main__': + import sys + sys.exit(main(sys.argv)) + + + + + + + + + diff --git a/http_server.py b/http_server.py new file mode 100644 index 0000000..fa9ce25 --- /dev/null +++ b/http_server.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +from flask import Flask +from flask import request +from flask import jsonify + +import rethinkdb as r + +from random import randint +from time import time + +import json + +app = Flask(__name__) + +def md5sum(filename): + try: + hash = md5() + with open(filename, "rb") as f: + for chunk in iter(lambda: f.read(128 * hash.block_size), b""): + hash.update(chunk) + return hash.hexdigest() + except: + return None + +@app.route("/", methods=["GET"]) +def index(): + return(''' + + + Munyal API + + +

Munyal private API

+ + + ''') + +@app.route("/upload", methods=["POST"]) +def upload(): + try: + r.connect( "localhost", 28015).repl() + cursor = r.table("changes") + + host = request.form.get("host") + action = request.form.get("action") + route = request.form.get("route") + obj = { + 'id' : str(time()).split('.')[0] + str(randint(1, 1000000)), + 'action': action, + 'route': route, + 'host': host + } + status = 'ok' + try: + cursor.insert(obj).run() + except: + status = 'error' + except: + status = 'error' + obj['status'] = status + return jsonify(obj) + + +if __name__ == '__main__': + app.run(debug=True) diff --git a/img/logo.png b/img/logo.png new file mode 100644 index 0000000..ab5f196 Binary files /dev/null and b/img/logo.png differ diff --git a/second_test.py b/second_test.py new file mode 100644 index 0000000..869acdd --- /dev/null +++ b/second_test.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +from compare_json import compare_json +from dir_to_json import get_json + +from copy import deepcopy +from time import sleep +from requests import post + +import os +import socket +import json +import ftplib + +def need_deleted(items, route): + out = [] + for item in items: + if item.get("is_file"): + out.append({"action": "delete", "route": os.path.join(route, item.get('name'))}) + else: + if item.get('content'): + out = out + need_deleted(item.get("content"), os.path.join(route, item.get('name'))) + else: + out.append({"action": "delete_folder", "route": os.path.join(route, item.get('name'))}) + return out + +def need_added(items, route): + out = [] + for item in items: + if item.get("is_file"): + out.append({"action": "add", "route": os.path.join(route, item.get('name'))}) + else: + if item.get('content'): + out = out + need_added(item.get("content"), os.path.join(route, item.get('name'))) + else: + out.append({"action": "add_folder", "route": os.path.join(route, item.get('name'))}) + return out + +def get_changes(jsons, route=''): + delete, add = jsons + out = need_deleted(delete, route) + need_added(add, route) + return out + ''' + out = [] + + delete, add = jsons + for item in delete: + if item.get("is_file"): + + if item.get("tag"): + if item.get("tag") == "update": + out.append({"action": "update", "file": os.path.join(route, item.get('name'))}) + elif item.get("tag") == "delete":##### Caso hipotetico imposible ##### + if item.get("is_file"): + out.append({"action": "delete", "file": os.path.join(route, item.get('name'))}) + else: + out.append({"action": "delete_folder", "file": os.path.join(route, item.get('name'))}) + + else: + out.append({"action": "delete", "file": os.path.join(route, item.get('name'))}) + + out.append({"action": "delete", "file": os.path.join(route, item.get('name'))}) + else: + return out + ''' + +ORIGINAL = "/home/kirbylife/Proyectos/munyal_test/original" + +def main(args): + ftp = ftplib.FTP('localhost', 'munyal', '123') + actual = get_json(ORIGINAL) + new = deepcopy(actual) + switch = lambda x,o,d=None: o.get(x) if o.get(x) else d if d else lambda *args: None + while True: + sleep(1) + jsons = compare_json(deepcopy(actual), deepcopy(new)) + changes = get_changes(jsons) + for change in changes: + route = os.path.join(ORIGINAL, change['route']) + success = False + while not success: + # ~ try: + x = change['route'] + if change['action'] == 'add': + print("Agregar archivo") + with open(route, "rb") as f: + ftp.storbinary("STOR /" + x, f) + elif change['action'] == 'add_folder': + print("Agregar carpeta") + ftp.mkd(x) + elif change['action'] == 'delete': + print("Borrar archivo") + ftp.delete(x) + elif change['action'] == 'delete_folder': + print("Borrar carpeta") + ftp.rmd(x) + else: + print("Unexpected action") + r = post('http://127.0.0.1:5000/upload', data={ + 'host': socket.gethostname(), + 'action': change['action'], + 'route': change['route'] + } + ) + r = json.loads(r.text) + print(json.dumps(r, indent=4)) + success = r['status'] == 'ok' + # ~ except: + # ~ print("Error uploading, retrying again\n") + actual = deepcopy(new) + new = get_json(ORIGINAL) + return 0 + +if __name__ == '__main__': + import sys + sys.exit(main(sys.argv))