diff --git a/client.py b/client.py
new file mode 100755
index 0000000..0b901d9
--- /dev/null
+++ b/client.py
@@ -0,0 +1,359 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+import ftplib
+import json
+import os
+import pathlib
+import socket
+import sys
+from copy import deepcopy
+from random import randint
+from shutil import rmtree
+from threading import Thread
+from time import sleep
+from tkinter import Button, Entry, Label, StringVar, Tk
+
+from PIL import Image, ImageTk
+from requests import post
+
+# import rethinkdb as r
+from compare_json import compare_json
+from dir_to_json import get_json
+
+# from tcping import Ping
+
+#ORIGINAL = "/home/kirbylife/Proyectos/munyal_test/another"
+ORIGINAL = "/home/kirbylife/Proyectos/munyal_test/"
+IP = "localhost"
+USERNAME = "munyal"
+PASSWORD = "123"
+HOSTNAME = socket.gethostname() + str(randint(1, 100000000))
+SKIP_UPLOAD = []
+FLAG_DOWNLOAD = False
+pending_routes = []
+
+
+def check_network(port):
+ ping = Ping(IP, port, 20)
+ ping.ping(1)
+ print(SKIP_UPLOAD)
+ return ping.result.rows[0].successed == 1
+
+
+def watch_dir():
+ global pending_routes
+ folder = os.path.join(os.getenv("HOME"), ".munyal")
+ if not os.path.exists(folder):
+ pathlib.Path(folder).mkdir(parents=True)
+
+ actual_file = os.path.join(folder, "actual.json")
+ if not os.path.exists(actual_file):
+ with open(actual_file, "w") as f:
+ f.write(json.dumps([]))
+ pending_file = os.path.join(folder, "pending_routes.json")
+ if not os.path.exists(pending_file):
+ with open(pending_file, "w") as f:
+ f.write(json.dumps([]))
+
+ with open(actual_file, "r") as f:
+ actual = json.loads(f.read())
+ actual = get_json(ORIGINAL)
+ new = deepcopy(actual)
+ with open(pending_file, "r") as f:
+ pending_routes = json.loads(f.read())
+ new = get_json(ORIGINAL)
+ while True:
+ sleep(0.2)
+ while True:
+ try:
+ jsons = compare_json(deepcopy(actual), deepcopy(new))
+ except:
+ new = get_json(ORIGINAL)
+ else:
+ break
+ changes = get_changes(jsons)
+
+ pending_routes = pending_routes + changes
+ with open(pending_file, "w") as f:
+ f.write(json.dumps(pending_routes, indent=4))
+
+ actual = deepcopy(new)
+ with open(actual_file, "w") as f:
+ f.write(json.dumps(actual, indent=4))
+ while True:
+ try:
+ new = get_json(ORIGINAL)
+ except:
+ pass
+ else:
+ break
+
+
+def need_deleted(items, route):
+ out = []
+ for item in items:
+ if item.get("is_file"):
+ out.append({
+ "action": "delete",
+ "route": os.path.join(route, item.get('name'))
+ })
+ else:
+ if item.get('content'):
+ out = out + need_deleted(item.get("content"),
+ os.path.join(route, item.get('name')))
+ else:
+ out.append({
+ "action": "delete_folder",
+ "route": os.path.join(route, item.get('name'))
+ })
+ return out
+
+
+def need_added(items, route):
+ out = []
+ for item in items:
+ if item.get("is_file"):
+ out.append({
+ "action": "add",
+ "route": os.path.join(route, item.get('name'))
+ })
+ else:
+ if item.get('content'):
+ out = out + need_added(item.get("content"),
+ os.path.join(route, item.get('name')))
+ else:
+ out.append({
+ "action": "add_folder",
+ "route": os.path.join(route, item.get('name'))
+ })
+ return out
+
+
+def get_changes(jsons, route=''):
+ delete, add = jsons
+ out = need_deleted(delete, route) + need_added(add, route)
+ return out
+
+
+def _is_ftp_dir(ftp_handle, name):
+ original_cwd = ftp_handle.pwd()
+ try:
+ ftp_handle.cwd(name)
+ ftp_handle.cwd(original_cwd)
+ return True
+ except:
+ return False
+
+
+def _make_parent_dir(fpath):
+ #dirname = os.path.dirname(fpath)
+ dirname = os.path.join(ORIGINAL, fpath)
+ while not os.path.exists(dirname):
+ try:
+ os.makedirs(dirname)
+ print("created {0}".format(dirname))
+ except:
+ _make_parent_dir(dirname)
+
+
+def _download_ftp_file(ftp_handle, name, dest, overwrite):
+ if not os.path.exists(dest) or overwrite is True:
+ try:
+ with open(dest, 'wb') as f:
+ ftp_handle.retrbinary("RETR {0}".format(name), f.write)
+ print("downloaded: {0}".format(dest))
+ except FileNotFoundError:
+ print("FAILED: {0}".format(dest))
+ else:
+ print("already exists: {0}".format(dest))
+
+
+def _mirror_ftp_dir(ftp_handle, name, overwrite):
+ for item in ftp_handle.nlst(name):
+ SKIP_UPLOAD.append(item)
+ if _is_ftp_dir(ftp_handle, item):
+ _make_parent_dir(item.lstrip("/"))
+ _mirror_ftp_dir(ftp_handle, os.path.join(name, item), overwrite)
+ else:
+ _download_ftp_file(ftp_handle, item, os.path.join(ORIGINAL, item),
+ overwrite)
+
+
+def download_ftp_tree(overwrite=False):
+ FLAG_DOWNLOAD = True
+ ftp_handle = ftplib.FTP(IP, USERNAME, PASSWORD)
+ path = ""
+ original_directory = os.getcwd()
+ os.chdir(ORIGINAL)
+ _mirror_ftp_dir(ftp_handle, path, overwrite)
+ os.chdir(original_directory)
+ ftp_handle.close()
+ FLAG_DOWNLOAD = False
+
+
+def upload(*args):
+ global SKIP_UPLOAD
+ global pending_routes
+ print("Modulo de subida listo")
+ while True:
+ sleep(0.1)
+ if check_network('21') and pending_routes:
+ change = pending_routes.pop(0)
+ if change['route'] not in SKIP_UPLOAD:
+ ftp = ftplib.FTP(IP, USERNAME, PASSWORD)
+ route = os.path.join(ORIGINAL, change['route'])
+ success = False
+ while not success:
+ try:
+ if change['action'] == 'add':
+ while FLAG_DOWNLOAD:
+ print("Wait")
+ print("Agregar archivo")
+ with open(route, "rb") as f:
+ ftp.storbinary("STOR /" + change['route'], f)
+ elif change['action'] == 'add_folder':
+ print("Agregar carpeta")
+ ftp.mkd(change['route'])
+ elif change['action'] == 'delete':
+ print("Borrar archivo")
+ ftp.delete(change['route'])
+ elif change['action'] == 'delete_folder':
+ print("Borrar carpeta")
+ ftp.rmd(change['route'])
+ else:
+ print("Unexpected action")
+ except:
+ print("Error uploading\n")
+ r = post("http://" + IP + ':8000/upload',
+ data={
+ 'host': HOSTNAME,
+ 'action': change['action'],
+ 'route': change['route']
+ })
+ r = json.loads(r.text)
+ print(json.dumps(r, indent=4))
+ success = r['status'] == 'ok'
+ ftp.close()
+ else:
+ SKIP_UPLOAD.pop()
+ return 0
+
+
+def download(*args):
+ global SKIP_UPLOAD
+ while True:
+ sleep(1)
+ if check_network(28015) and check_network(21):
+ try:
+ download_ftp_tree(overwrite=False)
+
+ print("Modulo de descarga listo")
+ print("Carpeta " + ORIGINAL)
+ r.connect(IP, 28015).repl()
+ cursor = r.table("changes").changes().run()
+ for document in cursor:
+ change = document['new_val']
+ #print(change)
+ if change['host'] != HOSTNAME:
+ route = os.path.join(ORIGINAL, change['route'])
+ SKIP_UPLOAD.append(change['route'])
+ try:
+ if change['action'] == 'add':
+ print("Agregar archivo")
+ FLAG_DOWNLOAD = True
+ ftp = ftplib.FTP(IP, USERNAME, PASSWORD)
+ with open(route, "wb") as f:
+ ftp.retrbinary("RETR /" + change['route'],
+ f.write)
+ ftp.close()
+ FLAG_DOWNLOAD = False
+ elif change['action'] == 'add_folder':
+ print("Agregar carpeta")
+ pathlib.Path(route).mkdir(parents=True)
+ elif change['action'] == 'delete':
+ print("Borrar archivo")
+ pathlib.Path(route).unlink()
+ elif change['action'] == 'delete_folder':
+ print("Borrar carpeta")
+ rmtree(route)
+ else:
+ print("Unexpected action")
+ except OSError as e:
+ print("Error en el sistema operativo")
+ except:
+ print("Error en el servidor FTP")
+ except r.errors.ReqlDriverError as e:
+ print("Conection refused with rethinkdb")
+ return 0
+
+
+def run_client(window, password, username, host, folder):
+ global PASSWORD
+ global IP
+ global USERNAME
+ global ORIGINAL
+
+ PASSWORD = password.get()
+ USERNAME = username.get()
+ IP = host.get()
+ ORIGINAL = folder.get()
+
+ if not os.path.exists(ORIGINAL):
+ pathlib.Path(ORIGINAL).mkdir(parents=True)
+
+ download_thread = Thread(target=download, args=[window])
+ download_thread.setDaemon(True)
+ download_thread.start()
+
+ upload_thread = Thread(target=upload, args=[window])
+ upload_thread.setDaemon(True)
+ upload_thread.start()
+
+ watch_dir_thread = Thread(target=watch_dir)
+ watch_dir_thread.setDaemon(True)
+ watch_dir_thread.start()
+
+
+def main(args):
+ root = Tk()
+ root.geometry("250x400")
+
+ img_logo = Image.open("img/logo.png")
+ img_tk = ImageTk.PhotoImage(img_logo.resize((200, 150), Image.ANTIALIAS))
+
+ host = StringVar()
+ host.set("localhost")
+ host_field = Entry(root, textvariable=host)
+
+ passwd = StringVar()
+ passwd_field = Entry(root, textvariable=passwd, show="*")
+
+ folder = StringVar()
+ folder.set(os.path.join(os.getenv("HOME"), "Munyal"))
+ folder_field = Entry(root, textvariable=folder)
+
+ connect = Button(root,
+ text="Conectar",
+ command=lambda: run_client(root, passwd, host, folder))
+
+ Label(root, text="MUNYAL").pack()
+ Label(root, image=img_tk).pack()
+ Label(root, text="").pack()
+ Label(root, text="Ruta del servidor").pack()
+ host_field.pack()
+ Label(root, text="").pack()
+ Label(root, text="Contraseña").pack()
+ passwd_field.pack()
+ Label(root, text="").pack()
+ Label(root, text="Carpeta a sincronizar").pack()
+ folder_field.pack()
+ Label(root, text="").pack()
+ connect.pack()
+
+ root.mainloop()
+
+
+if __name__ == '__main__':
+ import sys
+ sys.exit(main(sys.argv))
diff --git a/compare_json.py b/compare_json.py
new file mode 100644
index 0000000..77c1d15
--- /dev/null
+++ b/compare_json.py
@@ -0,0 +1,49 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+import json
+
+from dir_to_json import get_json
+
+def compare_json(json1, json2):
+ bckup1, bckup2 = json1[:], json2[:]
+ items1 = list(enumerate(json1))
+ items2 = list(enumerate(json2))
+ for i, item1 in items1:
+ for j, item2 in items2:
+ if item1["name"] == item2["name"]:
+ if item1["is_file"] == True == item2["is_file"]:
+ if item1["checksum"] == item2["checksum"]:
+ json1[i] = None
+ json2[j] = None
+ '''
+ else:
+ json1[i]["tag"] = "update"
+ json2[j] = None
+ '''
+ elif item1["is_file"] == False == item2["is_file"]:
+ new_json1, new_json2 = compare_json(item1["content"], item2["content"])
+ if len(new_json1) == 0:
+ json1[i] = None
+ else:
+ json1[i]["content"] = new_json1
+ if len(new_json2) == 0:
+ json2[j] = None
+ else:
+ json2[j]["content"] = new_json2
+ elif item1["is_file"] != item2["is_file"]:##### Caso hipotetico imposible #####
+ json1[i]["tag"] == "delete"
+ json1 = list(filter(None, json1))
+ json2 = list(filter(None, json2))
+ return json1, json2
+if __name__ == "__main__":
+ try:
+ json1 = get_json("/home/kirbylife/Proyectos/munyal_test/original")
+ json2 = get_json("/home/kirbylife/Proyectos/munyal_test/copy")
+ except:
+ print("error outside")
+ json1, json2 = compare_json(json1, json2)
+ #print(len(json1), len(json2))
+ print(json.dumps(json1, indent=4))
+ print("\n============\n")
+ print(json.dumps(json2, indent=4))
diff --git a/config.py b/config.py
new file mode 100644
index 0000000..6b7a7ec
--- /dev/null
+++ b/config.py
@@ -0,0 +1,43 @@
+import os
+from tkinter import Button, Entry, Label, StringVar, PhotoImage, Tk, Canvas
+
+
+def main(args):
+ root = Tk()
+ root.title("Munyal")
+ root.geometry("320x500")
+
+ logo = PhotoImage(file="img/logo.png")
+
+ host = StringVar()
+ host_field = Entry(root, textvariable=host, width=30)
+
+ passwd = StringVar()
+ passwd_field = Entry(root, textvariable=passwd, show="*", width=30)
+
+ folder = StringVar()
+ folder.set(os.path.join(os.getenv("HOME"), "Munyal"))
+ folder_field = Entry(root, textvariable=folder, width=30)
+
+ connect = Button(root, text="Conectar", command=lambda: None, width=10)
+
+ Label(root, image=logo).pack()
+ Label(root, text="MUNYAL", font=("", 30)).pack()
+ Label(root, text="").pack()
+ Label(root, text="Nombre del servidor").pack()
+ host_field.pack()
+ Label(root, text="").pack()
+ Label(root, text="Contraseña").pack()
+ passwd_field.pack()
+ Label(root, text="").pack()
+ Label(root, text="Carpeta a sincronizar").pack()
+ folder_field.pack()
+ Label(root, text="").pack()
+ connect.pack()
+
+ root.mainloop()
+
+
+if __name__ == '__main__':
+ import sys
+ sys.exit(main(sys.argv))
diff --git a/dir_to_json.py b/dir_to_json.py
new file mode 100644
index 0000000..5f59b1a
--- /dev/null
+++ b/dir_to_json.py
@@ -0,0 +1,50 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+from hashlib import md5
+
+import os
+import json
+
+
+def md5sum(filename):
+ try:
+ hash = md5()
+ with open(filename, "rb") as f:
+ for chunk in iter(lambda: f.read(128 * hash.block_size), b""):
+ hash.update(chunk)
+ return hash.hexdigest()
+ except:
+ return None
+
+def get_json(path):
+ out = []
+ items = os.listdir(path)
+ try:
+ for item in items:
+ if item[0] != ".":
+ item_json = {
+ "name": item
+ }
+ route = os.path.join(path, item)
+ if os.path.isdir(route):
+ item_json["is_file"] = False
+ item_json["content"] = get_json(route)
+ elif os.path.isfile(route):
+ item_json["is_file"] = True
+ item_json["size"] = os.path.getsize(route)
+ item_json["last_modified"] = os.path.getmtime(route)
+ item_json["created_at"] = os.path.getctime(route)
+ checksum = md5sum(route)
+ if checksum:
+ item_json["checksum"] = checksum
+ else:
+ item = None
+ out.append(item_json)
+ except:
+ return get_json(path)
+ return out
+
+if __name__ == "__main__":
+ output = get_json("/media/kirbylife/DATOS/Proyectos/PyCharmProjects/Munyal/folder_test")
+ print(json.dumps(output, indent=4))
diff --git a/first_test.py b/first_test.py
new file mode 100644
index 0000000..38d97ba
--- /dev/null
+++ b/first_test.py
@@ -0,0 +1,42 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+from compare_json import compare_json
+from dir_to_json import get_json
+
+from copy import deepcopy
+from time import sleep
+
+ORIGINAL = "/home/kirbylife/Proyectos/munyal_test/original"
+
+def main(args):
+ actual = get_json(ORIGINAL)
+ new = deepcopy(actual)
+
+ while True:
+ delete, add = compare_json(deepcopy(actual), deepcopy(new))
+ for item in delete:
+ if item.get("tag"):
+ if item.get("tag") == "update":
+ print("Actualizado el archivo {}".format(item.get("name")))
+ else:
+ print("borrado el archivo {}".format(item.get("name")))
+
+ for item in add:
+ print("Agregado el archivo {}".format(item.get("name")))
+ actual = deepcopy(new)
+ new = get_json(ORIGINAL)
+ return 0
+
+if __name__ == '__main__':
+ import sys
+ sys.exit(main(sys.argv))
+
+
+
+
+
+
+
+
+
diff --git a/http_server.py b/http_server.py
new file mode 100644
index 0000000..fa9ce25
--- /dev/null
+++ b/http_server.py
@@ -0,0 +1,67 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+from flask import Flask
+from flask import request
+from flask import jsonify
+
+import rethinkdb as r
+
+from random import randint
+from time import time
+
+import json
+
+app = Flask(__name__)
+
+def md5sum(filename):
+ try:
+ hash = md5()
+ with open(filename, "rb") as f:
+ for chunk in iter(lambda: f.read(128 * hash.block_size), b""):
+ hash.update(chunk)
+ return hash.hexdigest()
+ except:
+ return None
+
+@app.route("/", methods=["GET"])
+def index():
+ return('''
+ <html>
+ <head>
+ <title>Munyal API</title>
+ </head>
+ <body>
+ <h1>Munyal private API</h1>
+ </body>
+ </html>
+ ''')
+
+@app.route("/upload", methods=["POST"])
+def upload():
+ try:
+ r.connect( "localhost", 28015).repl()
+ cursor = r.table("changes")
+
+ host = request.form.get("host")
+ action = request.form.get("action")
+ route = request.form.get("route")
+ obj = {
+ 'id' : str(time()).split('.')[0] + str(randint(1, 1000000)),
+ 'action': action,
+ 'route': route,
+ 'host': host
+ }
+ status = 'ok'
+ try:
+ cursor.insert(obj).run()
+ except:
+ status = 'error'
+ except:
+ status = 'error'
+ obj['status'] = status
+ return jsonify(obj)
+
+
+if __name__ == '__main__':
+ app.run(debug=True)
diff --git a/img/logo.png b/img/logo.png
new file mode 100644
index 0000000..ab5f196
Binary files /dev/null and b/img/logo.png differ
diff --git a/second_test.py b/second_test.py
new file mode 100644
index 0000000..869acdd
--- /dev/null
+++ b/second_test.py
@@ -0,0 +1,117 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+from compare_json import compare_json
+from dir_to_json import get_json
+
+from copy import deepcopy
+from time import sleep
+from requests import post
+
+import os
+import socket
+import json
+import ftplib
+
+def need_deleted(items, route):
+ out = []
+ for item in items:
+ if item.get("is_file"):
+ out.append({"action": "delete", "route": os.path.join(route, item.get('name'))})
+ else:
+ if item.get('content'):
+ out = out + need_deleted(item.get("content"), os.path.join(route, item.get('name')))
+ else:
+ out.append({"action": "delete_folder", "route": os.path.join(route, item.get('name'))})
+ return out
+
+def need_added(items, route):
+ out = []
+ for item in items:
+ if item.get("is_file"):
+ out.append({"action": "add", "route": os.path.join(route, item.get('name'))})
+ else:
+ if item.get('content'):
+ out = out + need_added(item.get("content"), os.path.join(route, item.get('name')))
+ else:
+ out.append({"action": "add_folder", "route": os.path.join(route, item.get('name'))})
+ return out
+
+def get_changes(jsons, route=''):
+ delete, add = jsons
+ out = need_deleted(delete, route) + need_added(add, route)
+ return out
+ '''
+ out = []
+
+ delete, add = jsons
+ for item in delete:
+ if item.get("is_file"):
+
+ if item.get("tag"):
+ if item.get("tag") == "update":
+ out.append({"action": "update", "file": os.path.join(route, item.get('name'))})
+ elif item.get("tag") == "delete":##### Caso hipotetico imposible #####
+ if item.get("is_file"):
+ out.append({"action": "delete", "file": os.path.join(route, item.get('name'))})
+ else:
+ out.append({"action": "delete_folder", "file": os.path.join(route, item.get('name'))})
+
+ else:
+ out.append({"action": "delete", "file": os.path.join(route, item.get('name'))})
+
+ out.append({"action": "delete", "file": os.path.join(route, item.get('name'))})
+ else:
+ return out
+ '''
+
+ORIGINAL = "/home/kirbylife/Proyectos/munyal_test/original"
+
+def main(args):
+ ftp = ftplib.FTP('localhost', 'munyal', '123')
+ actual = get_json(ORIGINAL)
+ new = deepcopy(actual)
+ switch = lambda x,o,d=None: o.get(x) if o.get(x) else d if d else lambda *args: None
+ while True:
+ sleep(1)
+ jsons = compare_json(deepcopy(actual), deepcopy(new))
+ changes = get_changes(jsons)
+ for change in changes:
+ route = os.path.join(ORIGINAL, change['route'])
+ success = False
+ while not success:
+ # ~ try:
+ x = change['route']
+ if change['action'] == 'add':
+ print("Agregar archivo")
+ with open(route, "rb") as f:
+ ftp.storbinary("STOR /" + x, f)
+ elif change['action'] == 'add_folder':
+ print("Agregar carpeta")
+ ftp.mkd(x)
+ elif change['action'] == 'delete':
+ print("Borrar archivo")
+ ftp.delete(x)
+ elif change['action'] == 'delete_folder':
+ print("Borrar carpeta")
+ ftp.rmd(x)
+ else:
+ print("Unexpected action")
+ r = post('http://127.0.0.1:5000/upload', data={
+ 'host': socket.gethostname(),
+ 'action': change['action'],
+ 'route': change['route']
+ }
+ )
+ r = json.loads(r.text)
+ print(json.dumps(r, indent=4))
+ success = r['status'] == 'ok'
+ # ~ except:
+ # ~ print("Error uploading, retrying again\n")
+ actual = deepcopy(new)
+ new = get_json(ORIGINAL)
+ return 0
+
+if __name__ == '__main__':
+ import sys
+ sys.exit(main(sys.argv))