Initial commit

master
kirbylife 2019-07-28 11:18:49 -05:00
parent 9f3408c29b
commit f60b410124
8 changed files with 727 additions and 0 deletions

359
client.py 100755
View File

@ -0,0 +1,359 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import ftplib
import json
import os
import pathlib
import socket
import sys
from copy import deepcopy
from random import randint
from shutil import rmtree
from threading import Thread
from time import sleep
from tkinter import Button, Entry, Label, StringVar, Tk
from PIL import Image, ImageTk
from requests import post
# import rethinkdb as r
from compare_json import compare_json
from dir_to_json import get_json
# from tcping import Ping
#ORIGINAL = "/home/kirbylife/Proyectos/munyal_test/another"
ORIGINAL = "/home/kirbylife/Proyectos/munyal_test/"
IP = "localhost"
USERNAME = "munyal"
PASSWORD = "123"
HOSTNAME = socket.gethostname() + str(randint(1, 100000000))
SKIP_UPLOAD = []
FLAG_DOWNLOAD = False
pending_routes = []
def check_network(port):
ping = Ping(IP, port, 20)
ping.ping(1)
print(SKIP_UPLOAD)
return ping.result.rows[0].successed == 1
def watch_dir():
global pending_routes
folder = os.path.join(os.getenv("HOME"), ".munyal")
if not os.path.exists(folder):
pathlib.Path(folder).mkdir(parents=True)
actual_file = os.path.join(folder, "actual.json")
if not os.path.exists(actual_file):
with open(actual_file, "w") as f:
f.write(json.dumps([]))
pending_file = os.path.join(folder, "pending_routes.json")
if not os.path.exists(pending_file):
with open(pending_file, "w") as f:
f.write(json.dumps([]))
with open(actual_file, "r") as f:
actual = json.loads(f.read())
actual = get_json(ORIGINAL)
new = deepcopy(actual)
with open(pending_file, "r") as f:
pending_routes = json.loads(f.read())
new = get_json(ORIGINAL)
while True:
sleep(0.2)
while True:
try:
jsons = compare_json(deepcopy(actual), deepcopy(new))
except:
new = get_json(ORIGINAL)
else:
break
changes = get_changes(jsons)
pending_routes = pending_routes + changes
with open(pending_file, "w") as f:
f.write(json.dumps(pending_routes, indent=4))
actual = deepcopy(new)
with open(actual_file, "w") as f:
f.write(json.dumps(actual, indent=4))
while True:
try:
new = get_json(ORIGINAL)
except:
pass
else:
break
def need_deleted(items, route):
out = []
for item in items:
if item.get("is_file"):
out.append({
"action": "delete",
"route": os.path.join(route, item.get('name'))
})
else:
if item.get('content'):
out = out + need_deleted(item.get("content"),
os.path.join(route, item.get('name')))
else:
out.append({
"action": "delete_folder",
"route": os.path.join(route, item.get('name'))
})
return out
def need_added(items, route):
out = []
for item in items:
if item.get("is_file"):
out.append({
"action": "add",
"route": os.path.join(route, item.get('name'))
})
else:
if item.get('content'):
out = out + need_added(item.get("content"),
os.path.join(route, item.get('name')))
else:
out.append({
"action": "add_folder",
"route": os.path.join(route, item.get('name'))
})
return out
def get_changes(jsons, route=''):
delete, add = jsons
out = need_deleted(delete, route) + need_added(add, route)
return out
def _is_ftp_dir(ftp_handle, name):
original_cwd = ftp_handle.pwd()
try:
ftp_handle.cwd(name)
ftp_handle.cwd(original_cwd)
return True
except:
return False
def _make_parent_dir(fpath):
#dirname = os.path.dirname(fpath)
dirname = os.path.join(ORIGINAL, fpath)
while not os.path.exists(dirname):
try:
os.makedirs(dirname)
print("created {0}".format(dirname))
except:
_make_parent_dir(dirname)
def _download_ftp_file(ftp_handle, name, dest, overwrite):
if not os.path.exists(dest) or overwrite is True:
try:
with open(dest, 'wb') as f:
ftp_handle.retrbinary("RETR {0}".format(name), f.write)
print("downloaded: {0}".format(dest))
except FileNotFoundError:
print("FAILED: {0}".format(dest))
else:
print("already exists: {0}".format(dest))
def _mirror_ftp_dir(ftp_handle, name, overwrite):
for item in ftp_handle.nlst(name):
SKIP_UPLOAD.append(item)
if _is_ftp_dir(ftp_handle, item):
_make_parent_dir(item.lstrip("/"))
_mirror_ftp_dir(ftp_handle, os.path.join(name, item), overwrite)
else:
_download_ftp_file(ftp_handle, item, os.path.join(ORIGINAL, item),
overwrite)
def download_ftp_tree(overwrite=False):
FLAG_DOWNLOAD = True
ftp_handle = ftplib.FTP(IP, USERNAME, PASSWORD)
path = ""
original_directory = os.getcwd()
os.chdir(ORIGINAL)
_mirror_ftp_dir(ftp_handle, path, overwrite)
os.chdir(original_directory)
ftp_handle.close()
FLAG_DOWNLOAD = False
def upload(*args):
global SKIP_UPLOAD
global pending_routes
print("Modulo de subida listo")
while True:
sleep(0.1)
if check_network('21') and pending_routes:
change = pending_routes.pop(0)
if change['route'] not in SKIP_UPLOAD:
ftp = ftplib.FTP(IP, USERNAME, PASSWORD)
route = os.path.join(ORIGINAL, change['route'])
success = False
while not success:
try:
if change['action'] == 'add':
while FLAG_DOWNLOAD:
print("Wait")
print("Agregar archivo")
with open(route, "rb") as f:
ftp.storbinary("STOR /" + change['route'], f)
elif change['action'] == 'add_folder':
print("Agregar carpeta")
ftp.mkd(change['route'])
elif change['action'] == 'delete':
print("Borrar archivo")
ftp.delete(change['route'])
elif change['action'] == 'delete_folder':
print("Borrar carpeta")
ftp.rmd(change['route'])
else:
print("Unexpected action")
except:
print("Error uploading\n")
r = post("http://" + IP + ':8000/upload',
data={
'host': HOSTNAME,
'action': change['action'],
'route': change['route']
})
r = json.loads(r.text)
print(json.dumps(r, indent=4))
success = r['status'] == 'ok'
ftp.close()
else:
SKIP_UPLOAD.pop()
return 0
def download(*args):
global SKIP_UPLOAD
while True:
sleep(1)
if check_network(28015) and check_network(21):
try:
download_ftp_tree(overwrite=False)
print("Modulo de descarga listo")
print("Carpeta " + ORIGINAL)
r.connect(IP, 28015).repl()
cursor = r.table("changes").changes().run()
for document in cursor:
change = document['new_val']
#print(change)
if change['host'] != HOSTNAME:
route = os.path.join(ORIGINAL, change['route'])
SKIP_UPLOAD.append(change['route'])
try:
if change['action'] == 'add':
print("Agregar archivo")
FLAG_DOWNLOAD = True
ftp = ftplib.FTP(IP, USERNAME, PASSWORD)
with open(route, "wb") as f:
ftp.retrbinary("RETR /" + change['route'],
f.write)
ftp.close()
FLAG_DOWNLOAD = False
elif change['action'] == 'add_folder':
print("Agregar carpeta")
pathlib.Path(route).mkdir(parents=True)
elif change['action'] == 'delete':
print("Borrar archivo")
pathlib.Path(route).unlink()
elif change['action'] == 'delete_folder':
print("Borrar carpeta")
rmtree(route)
else:
print("Unexpected action")
except OSError as e:
print("Error en el sistema operativo")
except:
print("Error en el servidor FTP")
except r.errors.ReqlDriverError as e:
print("Conection refused with rethinkdb")
return 0
def run_client(window, password, username, host, folder):
global PASSWORD
global IP
global USERNAME
global ORIGINAL
PASSWORD = password.get()
USERNAME = username.get()
IP = host.get()
ORIGINAL = folder.get()
if not os.path.exists(ORIGINAL):
pathlib.Path(ORIGINAL).mkdir(parents=True)
download_thread = Thread(target=download, args=[window])
download_thread.setDaemon(True)
download_thread.start()
upload_thread = Thread(target=upload, args=[window])
upload_thread.setDaemon(True)
upload_thread.start()
watch_dir_thread = Thread(target=watch_dir)
watch_dir_thread.setDaemon(True)
watch_dir_thread.start()
def main(args):
root = Tk()
root.geometry("250x400")
img_logo = Image.open("img/logo.png")
img_tk = ImageTk.PhotoImage(img_logo.resize((200, 150), Image.ANTIALIAS))
host = StringVar()
host.set("localhost")
host_field = Entry(root, textvariable=host)
passwd = StringVar()
passwd_field = Entry(root, textvariable=passwd, show="*")
folder = StringVar()
folder.set(os.path.join(os.getenv("HOME"), "Munyal"))
folder_field = Entry(root, textvariable=folder)
connect = Button(root,
text="Conectar",
command=lambda: run_client(root, passwd, host, folder))
Label(root, text="MUNYAL").pack()
Label(root, image=img_tk).pack()
Label(root, text="").pack()
Label(root, text="Ruta del servidor").pack()
host_field.pack()
Label(root, text="").pack()
Label(root, text="Contraseña").pack()
passwd_field.pack()
Label(root, text="").pack()
Label(root, text="Carpeta a sincronizar").pack()
folder_field.pack()
Label(root, text="").pack()
connect.pack()
root.mainloop()
if __name__ == '__main__':
import sys
sys.exit(main(sys.argv))

49
compare_json.py 100644
View File

@ -0,0 +1,49 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import json
from dir_to_json import get_json
def compare_json(json1, json2):
bckup1, bckup2 = json1[:], json2[:]
items1 = list(enumerate(json1))
items2 = list(enumerate(json2))
for i, item1 in items1:
for j, item2 in items2:
if item1["name"] == item2["name"]:
if item1["is_file"] == True == item2["is_file"]:
if item1["checksum"] == item2["checksum"]:
json1[i] = None
json2[j] = None
'''
else:
json1[i]["tag"] = "update"
json2[j] = None
'''
elif item1["is_file"] == False == item2["is_file"]:
new_json1, new_json2 = compare_json(item1["content"], item2["content"])
if len(new_json1) == 0:
json1[i] = None
else:
json1[i]["content"] = new_json1
if len(new_json2) == 0:
json2[j] = None
else:
json2[j]["content"] = new_json2
elif item1["is_file"] != item2["is_file"]:##### Caso hipotetico imposible #####
json1[i]["tag"] == "delete"
json1 = list(filter(None, json1))
json2 = list(filter(None, json2))
return json1, json2
if __name__ == "__main__":
try:
json1 = get_json("/home/kirbylife/Proyectos/munyal_test/original")
json2 = get_json("/home/kirbylife/Proyectos/munyal_test/copy")
except:
print("error outside")
json1, json2 = compare_json(json1, json2)
#print(len(json1), len(json2))
print(json.dumps(json1, indent=4))
print("\n============\n")
print(json.dumps(json2, indent=4))

43
config.py 100644
View File

@ -0,0 +1,43 @@
import os
from tkinter import Button, Entry, Label, StringVar, PhotoImage, Tk, Canvas
def main(args):
root = Tk()
root.title("Munyal")
root.geometry("320x500")
logo = PhotoImage(file="img/logo.png")
host = StringVar()
host_field = Entry(root, textvariable=host, width=30)
passwd = StringVar()
passwd_field = Entry(root, textvariable=passwd, show="*", width=30)
folder = StringVar()
folder.set(os.path.join(os.getenv("HOME"), "Munyal"))
folder_field = Entry(root, textvariable=folder, width=30)
connect = Button(root, text="Conectar", command=lambda: None, width=10)
Label(root, image=logo).pack()
Label(root, text="MUNYAL", font=("", 30)).pack()
Label(root, text="").pack()
Label(root, text="Nombre del servidor").pack()
host_field.pack()
Label(root, text="").pack()
Label(root, text="Contraseña").pack()
passwd_field.pack()
Label(root, text="").pack()
Label(root, text="Carpeta a sincronizar").pack()
folder_field.pack()
Label(root, text="").pack()
connect.pack()
root.mainloop()
if __name__ == '__main__':
import sys
sys.exit(main(sys.argv))

50
dir_to_json.py 100644
View File

@ -0,0 +1,50 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
from hashlib import md5
import os
import json
def md5sum(filename):
try:
hash = md5()
with open(filename, "rb") as f:
for chunk in iter(lambda: f.read(128 * hash.block_size), b""):
hash.update(chunk)
return hash.hexdigest()
except:
return None
def get_json(path):
out = []
items = os.listdir(path)
try:
for item in items:
if item[0] != ".":
item_json = {
"name": item
}
route = os.path.join(path, item)
if os.path.isdir(route):
item_json["is_file"] = False
item_json["content"] = get_json(route)
elif os.path.isfile(route):
item_json["is_file"] = True
item_json["size"] = os.path.getsize(route)
item_json["last_modified"] = os.path.getmtime(route)
item_json["created_at"] = os.path.getctime(route)
checksum = md5sum(route)
if checksum:
item_json["checksum"] = checksum
else:
item = None
out.append(item_json)
except:
return get_json(path)
return out
if __name__ == "__main__":
output = get_json("/media/kirbylife/DATOS/Proyectos/PyCharmProjects/Munyal/folder_test")
print(json.dumps(output, indent=4))

42
first_test.py 100644
View File

@ -0,0 +1,42 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
from compare_json import compare_json
from dir_to_json import get_json
from copy import deepcopy
from time import sleep
ORIGINAL = "/home/kirbylife/Proyectos/munyal_test/original"
def main(args):
actual = get_json(ORIGINAL)
new = deepcopy(actual)
while True:
delete, add = compare_json(deepcopy(actual), deepcopy(new))
for item in delete:
if item.get("tag"):
if item.get("tag") == "update":
print("Actualizado el archivo {}".format(item.get("name")))
else:
print("borrado el archivo {}".format(item.get("name")))
for item in add:
print("Agregado el archivo {}".format(item.get("name")))
actual = deepcopy(new)
new = get_json(ORIGINAL)
return 0
if __name__ == '__main__':
import sys
sys.exit(main(sys.argv))

67
http_server.py 100644
View File

@ -0,0 +1,67 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
from flask import Flask
from flask import request
from flask import jsonify
import rethinkdb as r
from random import randint
from time import time
import json
app = Flask(__name__)
def md5sum(filename):
try:
hash = md5()
with open(filename, "rb") as f:
for chunk in iter(lambda: f.read(128 * hash.block_size), b""):
hash.update(chunk)
return hash.hexdigest()
except:
return None
@app.route("/", methods=["GET"])
def index():
return('''
<html>
<head>
<title>Munyal API</title>
</head>
<body>
<h1>Munyal private API</h1>
</body>
</html>
''')
@app.route("/upload", methods=["POST"])
def upload():
try:
r.connect( "localhost", 28015).repl()
cursor = r.table("changes")
host = request.form.get("host")
action = request.form.get("action")
route = request.form.get("route")
obj = {
'id' : str(time()).split('.')[0] + str(randint(1, 1000000)),
'action': action,
'route': route,
'host': host
}
status = 'ok'
try:
cursor.insert(obj).run()
except:
status = 'error'
except:
status = 'error'
obj['status'] = status
return jsonify(obj)
if __name__ == '__main__':
app.run(debug=True)

BIN
img/logo.png 100644

Binary file not shown.

After

Width:  |  Height:  |  Size: 42 KiB

117
second_test.py 100644
View File

@ -0,0 +1,117 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
from compare_json import compare_json
from dir_to_json import get_json
from copy import deepcopy
from time import sleep
from requests import post
import os
import socket
import json
import ftplib
def need_deleted(items, route):
out = []
for item in items:
if item.get("is_file"):
out.append({"action": "delete", "route": os.path.join(route, item.get('name'))})
else:
if item.get('content'):
out = out + need_deleted(item.get("content"), os.path.join(route, item.get('name')))
else:
out.append({"action": "delete_folder", "route": os.path.join(route, item.get('name'))})
return out
def need_added(items, route):
out = []
for item in items:
if item.get("is_file"):
out.append({"action": "add", "route": os.path.join(route, item.get('name'))})
else:
if item.get('content'):
out = out + need_added(item.get("content"), os.path.join(route, item.get('name')))
else:
out.append({"action": "add_folder", "route": os.path.join(route, item.get('name'))})
return out
def get_changes(jsons, route=''):
delete, add = jsons
out = need_deleted(delete, route) + need_added(add, route)
return out
'''
out = []
delete, add = jsons
for item in delete:
if item.get("is_file"):
if item.get("tag"):
if item.get("tag") == "update":
out.append({"action": "update", "file": os.path.join(route, item.get('name'))})
elif item.get("tag") == "delete":##### Caso hipotetico imposible #####
if item.get("is_file"):
out.append({"action": "delete", "file": os.path.join(route, item.get('name'))})
else:
out.append({"action": "delete_folder", "file": os.path.join(route, item.get('name'))})
else:
out.append({"action": "delete", "file": os.path.join(route, item.get('name'))})
out.append({"action": "delete", "file": os.path.join(route, item.get('name'))})
else:
return out
'''
ORIGINAL = "/home/kirbylife/Proyectos/munyal_test/original"
def main(args):
ftp = ftplib.FTP('localhost', 'munyal', '123')
actual = get_json(ORIGINAL)
new = deepcopy(actual)
switch = lambda x,o,d=None: o.get(x) if o.get(x) else d if d else lambda *args: None
while True:
sleep(1)
jsons = compare_json(deepcopy(actual), deepcopy(new))
changes = get_changes(jsons)
for change in changes:
route = os.path.join(ORIGINAL, change['route'])
success = False
while not success:
# ~ try:
x = change['route']
if change['action'] == 'add':
print("Agregar archivo")
with open(route, "rb") as f:
ftp.storbinary("STOR /" + x, f)
elif change['action'] == 'add_folder':
print("Agregar carpeta")
ftp.mkd(x)
elif change['action'] == 'delete':
print("Borrar archivo")
ftp.delete(x)
elif change['action'] == 'delete_folder':
print("Borrar carpeta")
ftp.rmd(x)
else:
print("Unexpected action")
r = post('http://127.0.0.1:5000/upload', data={
'host': socket.gethostname(),
'action': change['action'],
'route': change['route']
}
)
r = json.loads(r.text)
print(json.dumps(r, indent=4))
success = r['status'] == 'ok'
# ~ except:
# ~ print("Error uploading, retrying again\n")
actual = deepcopy(new)
new = get_json(ORIGINAL)
return 0
if __name__ == '__main__':
import sys
sys.exit(main(sys.argv))