reacondicioned all the code

master
kirbylife 2019-08-17 20:09:33 -05:00
parent 9bddcc1fa3
commit 0e6dde68c6
17 changed files with 765 additions and 340 deletions

373
client.py
View File

@ -1,359 +1,82 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
import ftplib
import json import json
import os import os
import pathlib import pathlib
import socket
import sys import sys
from copy import deepcopy from copy import deepcopy
from random import randint from random import randint
from shutil import rmtree from shutil import rmtree
from threading import Thread from threading import Thread
from time import sleep from time import sleep
from tkinter import Button, Entry, Label, StringVar, Tk from datetime import datetime
import pystray
from PIL import Image, ImageTk from PIL import Image, ImageTk
from requests import post from requests import post
from websocket import WebSocket
# import rethinkdb as r
from compare_json import compare_json from compare_json import compare_json
from config import get_config
from dir_to_json import get_json from dir_to_json import get_json
from misc import check_network
# from tcping import Ping
#ORIGINAL = "/home/kirbylife/Proyectos/munyal_test/another"
ORIGINAL = "/home/kirbylife/Proyectos/munyal_test/"
IP = "localhost"
USERNAME = "munyal"
PASSWORD = "123"
HOSTNAME = socket.gethostname() + str(randint(1, 100000000))
SKIP_UPLOAD = []
FLAG_DOWNLOAD = False
pending_routes = []
def check_network(port): class MunyalClient(pystray.Icon):
ping = Ping(IP, port, 20) def __init__(self):
ping.ping(1) super(MunyalClient, self).__init__("Munyal")
print(SKIP_UPLOAD) self.icon = Image.open("img/icons/standby.png")
return ping.result.rows[0].successed == 1 self.config = get_config()
self.stack = []
def is_online(self):
def watch_dir(): ping = check_network("http://google.com", 443)
global pending_routes if ping:
folder = os.path.join(os.getenv("HOME"), ".munyal") self.icon = Image.open("img/icons/online.png")
if not os.path.exists(folder):
pathlib.Path(folder).mkdir(parents=True)
actual_file = os.path.join(folder, "actual.json")
if not os.path.exists(actual_file):
with open(actual_file, "w") as f:
f.write(json.dumps([]))
pending_file = os.path.join(folder, "pending_routes.json")
if not os.path.exists(pending_file):
with open(pending_file, "w") as f:
f.write(json.dumps([]))
with open(actual_file, "r") as f:
actual = json.loads(f.read())
actual = get_json(ORIGINAL)
new = deepcopy(actual)
with open(pending_file, "r") as f:
pending_routes = json.loads(f.read())
new = get_json(ORIGINAL)
while True:
sleep(0.2)
while True:
try:
jsons = compare_json(deepcopy(actual), deepcopy(new))
except:
new = get_json(ORIGINAL)
else: else:
break self.icon = Image.open("img/icons/offline.png")
changes = get_changes(jsons)
pending_routes = pending_routes + changes def start(self):
with open(pending_file, "w") as f: thread_uploader = Thread(target=self.uploader, name="uploader", daemon=True)
f.write(json.dumps(pending_routes, indent=4)) thread_downloader = Thread(target=self.downloader, name="downloader", daemon=True)
thread_uploader.start()
thread_downloader.start()
self.run(self.__run)
actual = deepcopy(new) def __run(self, icon):
with open(actual_file, "w") as f: print("_run")
f.write(json.dumps(actual, indent=4)) icon.visible = True
while True: sleep(60)
try: self.stop()
new = get_json(ORIGINAL)
except: def listener(self):
pass pass
else:
break
def uploader(self):
def need_deleted(items, route): print("Uploader")
out = []
for item in items:
if item.get("is_file"):
out.append({
"action": "delete",
"route": os.path.join(route, item.get('name'))
})
else:
if item.get('content'):
out = out + need_deleted(item.get("content"),
os.path.join(route, item.get('name')))
else:
out.append({
"action": "delete_folder",
"route": os.path.join(route, item.get('name'))
})
return out
def need_added(items, route):
out = []
for item in items:
if item.get("is_file"):
out.append({
"action": "add",
"route": os.path.join(route, item.get('name'))
})
else:
if item.get('content'):
out = out + need_added(item.get("content"),
os.path.join(route, item.get('name')))
else:
out.append({
"action": "add_folder",
"route": os.path.join(route, item.get('name'))
})
return out
def get_changes(jsons, route=''):
delete, add = jsons
out = need_deleted(delete, route) + need_added(add, route)
return out
def _is_ftp_dir(ftp_handle, name):
original_cwd = ftp_handle.pwd()
try:
ftp_handle.cwd(name)
ftp_handle.cwd(original_cwd)
return True
except:
return False
def _make_parent_dir(fpath):
#dirname = os.path.dirname(fpath)
dirname = os.path.join(ORIGINAL, fpath)
while not os.path.exists(dirname):
try:
os.makedirs(dirname)
print("created {0}".format(dirname))
except:
_make_parent_dir(dirname)
def _download_ftp_file(ftp_handle, name, dest, overwrite):
if not os.path.exists(dest) or overwrite is True:
try:
with open(dest, 'wb') as f:
ftp_handle.retrbinary("RETR {0}".format(name), f.write)
print("downloaded: {0}".format(dest))
except FileNotFoundError:
print("FAILED: {0}".format(dest))
else:
print("already exists: {0}".format(dest))
def _mirror_ftp_dir(ftp_handle, name, overwrite):
for item in ftp_handle.nlst(name):
SKIP_UPLOAD.append(item)
if _is_ftp_dir(ftp_handle, item):
_make_parent_dir(item.lstrip("/"))
_mirror_ftp_dir(ftp_handle, os.path.join(name, item), overwrite)
else:
_download_ftp_file(ftp_handle, item, os.path.join(ORIGINAL, item),
overwrite)
def download_ftp_tree(overwrite=False):
FLAG_DOWNLOAD = True
ftp_handle = ftplib.FTP(IP, USERNAME, PASSWORD)
path = ""
original_directory = os.getcwd()
os.chdir(ORIGINAL)
_mirror_ftp_dir(ftp_handle, path, overwrite)
os.chdir(original_directory)
ftp_handle.close()
FLAG_DOWNLOAD = False
def upload(*args):
global SKIP_UPLOAD
global pending_routes
print("Modulo de subida listo")
while True: while True:
sleep(0.1) sleep(5)
if check_network('21') and pending_routes: self.icon = Image.open("img/icons/online.png")
change = pending_routes.pop(0)
if change['route'] not in SKIP_UPLOAD:
ftp = ftplib.FTP(IP, USERNAME, PASSWORD)
route = os.path.join(ORIGINAL, change['route'])
success = False
while not success:
try:
if change['action'] == 'add':
while FLAG_DOWNLOAD:
print("Wait")
print("Agregar archivo")
with open(route, "rb") as f:
ftp.storbinary("STOR /" + change['route'], f)
elif change['action'] == 'add_folder':
print("Agregar carpeta")
ftp.mkd(change['route'])
elif change['action'] == 'delete':
print("Borrar archivo")
ftp.delete(change['route'])
elif change['action'] == 'delete_folder':
print("Borrar carpeta")
ftp.rmd(change['route'])
else:
print("Unexpected action")
except:
print("Error uploading\n")
r = post("http://" + IP + ':8000/upload',
data={
'host': HOSTNAME,
'action': change['action'],
'route': change['route']
})
r = json.loads(r.text)
print(json.dumps(r, indent=4))
success = r['status'] == 'ok'
ftp.close()
else:
SKIP_UPLOAD.pop()
return 0
def download(*args): def __upload(self, path):
global SKIP_UPLOAD pass
def downloader(self):
print("Downloader")
while True: while True:
sleep(1) sleep(7)
if check_network(28015) and check_network(21): self.icon = Image.open("img/icons/offline.png")
try:
download_ftp_tree(overwrite=False)
print("Modulo de descarga listo") def __download(self, path):
print("Carpeta " + ORIGINAL) pass
r.connect(IP, 28015).repl()
cursor = r.table("changes").changes().run()
for document in cursor:
change = document['new_val']
#print(change)
if change['host'] != HOSTNAME:
route = os.path.join(ORIGINAL, change['route'])
SKIP_UPLOAD.append(change['route'])
try:
if change['action'] == 'add':
print("Agregar archivo")
FLAG_DOWNLOAD = True
ftp = ftplib.FTP(IP, USERNAME, PASSWORD)
with open(route, "wb") as f:
ftp.retrbinary("RETR /" + change['route'],
f.write)
ftp.close()
FLAG_DOWNLOAD = False
elif change['action'] == 'add_folder':
print("Agregar carpeta")
pathlib.Path(route).mkdir(parents=True)
elif change['action'] == 'delete':
print("Borrar archivo")
pathlib.Path(route).unlink()
elif change['action'] == 'delete_folder':
print("Borrar carpeta")
rmtree(route)
else:
print("Unexpected action")
except OSError as e:
print("Error en el sistema operativo")
except:
print("Error en el servidor FTP")
except r.errors.ReqlDriverError as e:
print("Conection refused with rethinkdb")
return 0
def run_client(window, password, username, host, folder):
global PASSWORD
global IP
global USERNAME
global ORIGINAL
PASSWORD = password.get()
USERNAME = username.get()
IP = host.get()
ORIGINAL = folder.get()
if not os.path.exists(ORIGINAL):
pathlib.Path(ORIGINAL).mkdir(parents=True)
download_thread = Thread(target=download, args=[window])
download_thread.setDaemon(True)
download_thread.start()
upload_thread = Thread(target=upload, args=[window])
upload_thread.setDaemon(True)
upload_thread.start()
watch_dir_thread = Thread(target=watch_dir)
watch_dir_thread.setDaemon(True)
watch_dir_thread.start()
def main(args):
root = Tk()
root.geometry("250x400")
img_logo = Image.open("img/logo.png")
img_tk = ImageTk.PhotoImage(img_logo.resize((200, 150), Image.ANTIALIAS))
host = StringVar()
host.set("localhost")
host_field = Entry(root, textvariable=host)
passwd = StringVar()
passwd_field = Entry(root, textvariable=passwd, show="*")
folder = StringVar()
folder.set(os.path.join(os.getenv("HOME"), "Munyal"))
folder_field = Entry(root, textvariable=folder)
connect = Button(root,
text="Conectar",
command=lambda: run_client(root, passwd, host, folder))
Label(root, text="MUNYAL").pack()
Label(root, image=img_tk).pack()
Label(root, text="").pack()
Label(root, text="Ruta del servidor").pack()
host_field.pack()
Label(root, text="").pack()
Label(root, text="Contraseña").pack()
passwd_field.pack()
Label(root, text="").pack()
Label(root, text="Carpeta a sincronizar").pack()
folder_field.pack()
Label(root, text="").pack()
connect.pack()
root.mainloop()
if __name__ == '__main__': if __name__ == '__main__':
import sys try:
sys.exit(main(sys.argv)) client = MunyalClient()
client.start()
sys.exit(0)
except BaseException:
sys.exit(1)

View File

@ -1,19 +1,18 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
from flask import Flask import json
from flask import request from hashlib import md5
from flask import jsonify
import rethinkdb as r
from random import randint from random import randint
from time import time from time import time
import json from flask import Flask, jsonify, request
import rethinkdb as r
app = Flask(__name__) app = Flask(__name__)
def md5sum(filename): def md5sum(filename):
try: try:
hash = md5() hash = md5()
@ -24,9 +23,10 @@ def md5sum(filename):
except: except:
return None return None
@app.route("/", methods=["GET"]) @app.route("/", methods=["GET"])
def index(): def index():
return(''' return ('''
<html> <html>
<head> <head>
<title>Munyal API</title> <title>Munyal API</title>
@ -37,17 +37,18 @@ def index():
</html> </html>
''') ''')
@app.route("/upload", methods=["POST"]) @app.route("/upload", methods=["POST"])
def upload(): def upload():
try: try:
r.connect( "localhost", 28015).repl() r.connect("localhost", 28015).repl()
cursor = r.table("changes") cursor = r.table("changes")
host = request.form.get("host") host = request.form.get("host")
action = request.form.get("action") action = request.form.get("action")
route = request.form.get("route") route = request.form.get("route")
obj = { obj = {
'id' : str(time()).split('.')[0] + str(randint(1, 1000000)), 'id': str(time()).split('.')[0] + str(randint(1, 1000000)),
'action': action, 'action': action,
'route': route, 'route': route,
'host': host 'host': host

Binary file not shown.

After

Width:  |  Height:  |  Size: 5.9 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 116 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 5.9 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 116 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 5.9 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 116 KiB

13
main.py 100644
View File

@ -0,0 +1,13 @@
import os
import sys
from client import MunyalClient
from misc import get_os
def main():
os = get_os()
if __name__ == "__main__":
sys.exit(main())

View File

@ -8,7 +8,6 @@ from tcping import Ping
def check_network(ip, port): def check_network(ip, port):
return True
ping = Ping(ip, port, 20) ping = Ping(ip, port, 20)
ping.ping(1) ping.ping(1)
return ping.result.rows[0].successed == 1 return ping.result.rows[0].successed == 1

363
temp/client.py 100755
View File

@ -0,0 +1,363 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import ftplib
import json
import os
import pathlib
import socket
import sys
from copy import deepcopy
from random import randint
from shutil import rmtree
from threading import Thread
from time import sleep
from tkinter import Button, Entry, Label, StringVar, Tk
from requests import post
import rethinkdb as r
from compare_json import compare_json
from dir_to_json import get_json
from tcping import Ping
#ORIGINAL = "/home/kirbylife/Proyectos/munyal_test/another"
ORIGINAL = "/home/kirbylife/Proyectos/munyal_test/"
IP = "localhost"
USERNAME = "munyal"
PASSWORD = "123"
HOSTNAME = socket.gethostname() + str(randint(1, 100000000))
SKIP_UPLOAD = []
FLAG_DOWNLOAD = False
pending_routes = []
def check_network(port):
ping = Ping(IP, port, 20)
ping.ping(1)
print(SKIP_UPLOAD)
return ping.result.rows[0].successed == 1
def watch_dir():
global pending_routes
folder = os.path.join(os.getenv("HOME"), ".munyal")
if not os.path.exists(folder):
pathlib.Path(folder).mkdir(parents=True)
actual_file = os.path.join(folder, "actual.json")
if not os.path.exists(actual_file):
with open(actual_file, "w") as f:
f.write(json.dumps([]))
pending_file = os.path.join(folder, "pending_routes.json")
if not os.path.exists(pending_file):
with open(pending_file, "w") as f:
f.write(json.dumps([]))
with open(actual_file, "r") as f:
actual = json.loads(f.read())
actual = get_json(ORIGINAL)
new = deepcopy(actual)
with open(pending_file, "r") as f:
pending_routes = json.loads(f.read())
new = get_json(ORIGINAL)
while True:
sleep(0.2)
while True:
try:
jsons = compare_json(deepcopy(actual), deepcopy(new))
except:
new = get_json(ORIGINAL)
else:
break
changes = get_changes(jsons)
pending_routes = pending_routes + changes
with open(pending_file, "w") as f:
f.write(json.dumps(pending_routes, indent=4))
actual = deepcopy(new)
with open(actual_file, "w") as f:
f.write(json.dumps(actual, indent=4))
while True:
try:
new = get_json(ORIGINAL)
except:
pass
else:
break
def need_deleted(items, route):
out = []
for item in items:
if item.get("is_file"):
out.append({
"action": "delete",
"route": os.path.join(route, item.get('name'))
})
else:
if item.get('content'):
out = out + need_deleted(item.get("content"),
os.path.join(route, item.get('name')))
else:
out.append({
"action": "delete_folder",
"route": os.path.join(route, item.get('name'))
})
return out
def need_added(items, route):
out = []
for item in items:
if item.get("is_file"):
out.append({
"action": "add",
"route": os.path.join(route, item.get('name'))
})
else:
if item.get('content'):
out = out + need_added(item.get("content"),
os.path.join(route, item.get('name')))
else:
out.append({
"action": "add_folder",
"route": os.path.join(route, item.get('name'))
})
return out
def get_changes(jsons, route=''):
delete, add = jsons
out = need_deleted(delete, route) + need_added(add, route)
return out
def _is_ftp_dir(ftp_handle, name):
original_cwd = ftp_handle.pwd()
try:
ftp_handle.cwd(name)
ftp_handle.cwd(original_cwd)
return True
except:
return False
def _make_parent_dir(fpath):
#dirname = os.path.dirname(fpath)
dirname = os.path.join(ORIGINAL, fpath)
while not os.path.exists(dirname):
try:
os.makedirs(dirname)
print("created {0}".format(dirname))
except:
_make_parent_dir(dirname)
def _download_ftp_file(ftp_handle, name, dest, overwrite):
if not os.path.exists(dest) or overwrite is True:
try:
with open(dest, 'wb') as f:
ftp_handle.retrbinary("RETR {0}".format(name), f.write)
print("downloaded: {0}".format(dest))
except FileNotFoundError:
print("FAILED: {0}".format(dest))
else:
print("already exists: {0}".format(dest))
def _mirror_ftp_dir(ftp_handle, name, overwrite):
for item in ftp_handle.nlst(name):
SKIP_UPLOAD.append(item)
if _is_ftp_dir(ftp_handle, item):
_make_parent_dir(item.lstrip("/"))
_mirror_ftp_dir(ftp_handle, os.path.join(name, item), overwrite)
else:
_download_ftp_file(ftp_handle, item, os.path.join(ORIGINAL, item),
overwrite)
def download_ftp_tree(overwrite=False):
FLAG_DOWNLOAD = True
ftp_handle = ftplib.FTP(IP, USERNAME, PASSWORD)
path = ""
original_directory = os.getcwd()
os.chdir(ORIGINAL)
_mirror_ftp_dir(ftp_handle, path, overwrite)
os.chdir(original_directory)
ftp_handle.close()
FLAG_DOWNLOAD = False
def upload(*args):
global SKIP_UPLOAD
global pending_routes
print("Modulo de subida listo")
while True:
sleep(0.1)
if check_network('21') and pending_routes:
change = pending_routes.pop(0)
while FLAG_DOWNLOAD:
print("Wait")
if change['route'] not in SKIP_UPLOAD:
ftp = ftplib.FTP(IP, USERNAME, PASSWORD)
route = os.path.join(ORIGINAL, change['route'])
success = False
while not success:
try:
if change['action'] == 'add':
print("Agregar archivo")
with open(route, "rb") as f:
ftp.storbinary("STOR /" + change['route'], f)
elif change['action'] == 'add_folder':
print("Agregar carpeta")
ftp.mkd(change['route'])
elif change['action'] == 'delete':
print("Borrar archivo")
ftp.delete(change['route'])
elif change['action'] == 'delete_folder':
print("Borrar carpeta")
ftp.rmd(change['route'])
else:
print("Unexpected action")
except:
print("Error uploading\n")
r = post("http://" + IP + ':5000/upload',
data={
'host': HOSTNAME,
'action': change['action'],
'route': change['route']
})
r = json.loads(r.text)
print(json.dumps(r, indent=4))
success = r['status'] == 'ok'
ftp.close()
else:
SKIP_UPLOAD.pop()
return 0
def download(*args):
global SKIP_UPLOAD
while True:
sleep(1)
if check_network(28015) and check_network(21):
try:
download_ftp_tree(overwrite=False)
print("Modulo de descarga listo")
print("Carpeta " + ORIGINAL)
r.connect(IP, 28015).repl()
cursor = r.table("changes").changes().run()
for document in cursor:
change = document['new_val']
#print(change)
if change['host'] != HOSTNAME:
FLAG_DOWNLOAD = True
route = os.path.join(ORIGINAL, change['route'])
SKIP_UPLOAD.append(change['route'])
try:
if change['action'] == 'add':
print("Agregar archivo")
ftp = ftplib.FTP(IP, USERNAME, PASSWORD)
with open(route, "wb") as f:
ftp.retrbinary("RETR /" + change['route'],
f.write)
ftp.close()
elif change['action'] == 'add_folder':
print("Agregar carpeta")
pathlib.Path(route).mkdir(parents=True)
elif change['action'] == 'delete':
print("Borrar archivo")
pathlib.Path(route).unlink()
elif change['action'] == 'delete_folder':
print("Borrar carpeta")
rmtree(route)
else:
print("Unexpected action")
except OSError as e:
print("Error en el sistema operativo")
except:
print("Error en el servidor FTP")
FLAG_DOWNLOAD = False
except rethinkdb.errors.ReqlDriverError as e:
print("Conection refused with rethinkdb")
return 0
def run_client(window, password, username, host, folder):
global PASSWORD
global IP
global USERNAME
global ORIGINAL
PASSWORD = password.get()
USERNAME = username.get()
IP = host.get()
ORIGINAL = folder.get()
if not os.path.exists(ORIGINAL):
pathlib.Path(ORIGINAL).mkdir(parents=True)
download_thread = Thread(target=download, args=[window])
download_thread.setDaemon(True)
download_thread.start()
upload_thread = Thread(target=upload, args=[window])
upload_thread.setDaemon(True)
upload_thread.start()
watch_dir_thread = Thread(target=watch_dir)
watch_dir_thread.setDaemon(True)
watch_dir_thread.start()
def main(args):
root = Tk()
root.geometry("200x300")
root.title("MUNYAL")
host = StringVar()
host.set("localhost")
host_field = Entry(root, textvariable=host)
user = StringVar()
user.set("munyal")
user_field = Entry(root, textvariable=user)
passwd = StringVar()
passwd.set("123")
passwd_field = Entry(root, textvariable=passwd, show="*")
folder = StringVar()
folder.set(os.path.join(os.getenv("HOME"), "Munyal"))
folder_field = Entry(root, textvariable=folder)
connect = Button(
root,
text="Conectar",
command=lambda: run_client(root, passwd, user, host, folder))
Label(root, text="MUNYAL").pack()
Label(root, text="").pack()
Label(root, text="Ruta del servidor").pack()
host_field.pack()
Label(root, text="").pack()
Label(root, text="Nombre de usuario").pack()
user_field.pack()
Label(root, text="").pack()
Label(root, text="Contraseña").pack()
passwd_field.pack()
Label(root, text="").pack()
Label(root, text="Carpeta a sincronizar").pack()
folder_field.pack()
Label(root, text="").pack()
connect.pack()
root.mainloop()
if __name__ == '__main__':
import sys
sys.exit(main(sys.argv))

View File

@ -0,0 +1,49 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import json
from dir_to_json import get_json
def compare_json(json1, json2):
bckup1, bckup2 = json1[:], json2[:]
items1 = list(enumerate(json1))
items2 = list(enumerate(json2))
for i, item1 in items1:
for j, item2 in items2:
if item1["name"] == item2["name"]:
if item1["is_file"] == True == item2["is_file"]:
if item1["checksum"] == item2["checksum"]:
json1[i] = None
json2[j] = None
'''
else:
json1[i]["tag"] = "update"
json2[j] = None
'''
elif item1["is_file"] == False == item2["is_file"]:
new_json1, new_json2 = compare_json(item1["content"], item2["content"])
if len(new_json1) == 0:
json1[i] = None
else:
json1[i]["content"] = new_json1
if len(new_json2) == 0:
json2[j] = None
else:
json2[j]["content"] = new_json2
elif item1["is_file"] != item2["is_file"]:##### Caso hipotetico imposible #####
json1[i]["tag"] == "delete"
json1 = list(filter(None, json1))
json2 = list(filter(None, json2))
return json1, json2
if __name__ == "__main__":
try:
json1 = get_json("/home/kirbylife/Proyectos/munyal_test/original")
json2 = get_json("/home/kirbylife/Proyectos/munyal_test/copy")
except:
print("error outside")
json1, json2 = compare_json(json1, json2)
#print(len(json1), len(json2))
print(json.dumps(json1, indent=4))
print("\n============\n")
print(json.dumps(json2, indent=4))

View File

@ -0,0 +1,50 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
from hashlib import md5
import os
import json
def md5sum(filename):
try:
hash = md5()
with open(filename, "rb") as f:
for chunk in iter(lambda: f.read(128 * hash.block_size), b""):
hash.update(chunk)
return hash.hexdigest()
except:
return None
def get_json(path):
out = []
items = os.listdir(path)
try:
for item in items:
if item[0] != ".":
item_json = {
"name": item
}
route = os.path.join(path, item)
if os.path.isdir(route):
item_json["is_file"] = False
item_json["content"] = get_json(route)
elif os.path.isfile(route):
item_json["is_file"] = True
item_json["size"] = os.path.getsize(route)
item_json["last_modified"] = os.path.getmtime(route)
item_json["created_at"] = os.path.getctime(route)
checksum = md5sum(route)
if checksum:
item_json["checksum"] = checksum
else:
item = None
out.append(item_json)
except:
return get_json(path)
return out
if __name__ == "__main__":
output = get_json("/media/kirbylife/DATOS/Proyectos/PyCharmProjects/Munyal/folder_test")
print(json.dumps(output, indent=4))

View File

@ -0,0 +1 @@

42
temp/first_test.py 100644
View File

@ -0,0 +1,42 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
from compare_json import compare_json
from dir_to_json import get_json
from copy import deepcopy
from time import sleep
ORIGINAL = "/home/kirbylife/Proyectos/munyal_test/original"
def main(args):
actual = get_json(ORIGINAL)
new = deepcopy(actual)
while True:
delete, add = compare_json(deepcopy(actual), deepcopy(new))
for item in delete:
if item.get("tag"):
if item.get("tag") == "update":
print("Actualizado el archivo {}".format(item.get("name")))
else:
print("borrado el archivo {}".format(item.get("name")))
for item in add:
print("Agregado el archivo {}".format(item.get("name")))
actual = deepcopy(new)
new = get_json(ORIGINAL)
return 0
if __name__ == '__main__':
import sys
sys.exit(main(sys.argv))

View File

@ -0,0 +1,67 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
from flask import Flask
from flask import request
from flask import jsonify
import rethinkdb as r
from random import randint
from time import time
import json
app = Flask(__name__)
def md5sum(filename):
try:
hash = md5()
with open(filename, "rb") as f:
for chunk in iter(lambda: f.read(128 * hash.block_size), b""):
hash.update(chunk)
return hash.hexdigest()
except:
return None
@app.route("/", methods=["GET"])
def index():
return('''
<html>
<head>
<title>Munyal API</title>
</head>
<body>
<h1>Munyal private API</h1>
</body>
</html>
''')
@app.route("/upload", methods=["POST"])
def upload():
try:
r.connect( "localhost", 28015).repl()
cursor = r.table("changes")
host = request.form.get("host")
action = request.form.get("action")
route = request.form.get("route")
obj = {
'id' : str(time()).split('.')[0] + str(randint(1, 1000000)),
'action': action,
'route': route,
'host': host
}
status = 'ok'
try:
cursor.insert(obj).run()
except:
status = 'error'
except:
status = 'error'
obj['status'] = status
return jsonify(obj)
if __name__ == '__main__':
app.run(debug=True)

117
temp/second_test.py 100644
View File

@ -0,0 +1,117 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
from compare_json import compare_json
from dir_to_json import get_json
from copy import deepcopy
from time import sleep
from requests import post
import os
import socket
import json
import ftplib
def need_deleted(items, route):
out = []
for item in items:
if item.get("is_file"):
out.append({"action": "delete", "route": os.path.join(route, item.get('name'))})
else:
if item.get('content'):
out = out + need_deleted(item.get("content"), os.path.join(route, item.get('name')))
else:
out.append({"action": "delete_folder", "route": os.path.join(route, item.get('name'))})
return out
def need_added(items, route):
out = []
for item in items:
if item.get("is_file"):
out.append({"action": "add", "route": os.path.join(route, item.get('name'))})
else:
if item.get('content'):
out = out + need_added(item.get("content"), os.path.join(route, item.get('name')))
else:
out.append({"action": "add_folder", "route": os.path.join(route, item.get('name'))})
return out
def get_changes(jsons, route=''):
delete, add = jsons
out = need_deleted(delete, route) + need_added(add, route)
return out
'''
out = []
delete, add = jsons
for item in delete:
if item.get("is_file"):
if item.get("tag"):
if item.get("tag") == "update":
out.append({"action": "update", "file": os.path.join(route, item.get('name'))})
elif item.get("tag") == "delete":##### Caso hipotetico imposible #####
if item.get("is_file"):
out.append({"action": "delete", "file": os.path.join(route, item.get('name'))})
else:
out.append({"action": "delete_folder", "file": os.path.join(route, item.get('name'))})
else:
out.append({"action": "delete", "file": os.path.join(route, item.get('name'))})
out.append({"action": "delete", "file": os.path.join(route, item.get('name'))})
else:
return out
'''
ORIGINAL = "/home/kirbylife/Proyectos/munyal_test/original"
def main(args):
ftp = ftplib.FTP('localhost', 'munyal', '123')
actual = get_json(ORIGINAL)
new = deepcopy(actual)
switch = lambda x,o,d=None: o.get(x) if o.get(x) else d if d else lambda *args: None
while True:
sleep(1)
jsons = compare_json(deepcopy(actual), deepcopy(new))
changes = get_changes(jsons)
for change in changes:
route = os.path.join(ORIGINAL, change['route'])
success = False
while not success:
# ~ try:
x = change['route']
if change['action'] == 'add':
print("Agregar archivo")
with open(route, "rb") as f:
ftp.storbinary("STOR /" + x, f)
elif change['action'] == 'add_folder':
print("Agregar carpeta")
ftp.mkd(x)
elif change['action'] == 'delete':
print("Borrar archivo")
ftp.delete(x)
elif change['action'] == 'delete_folder':
print("Borrar carpeta")
ftp.rmd(x)
else:
print("Unexpected action")
r = post('http://127.0.0.1:5000/upload', data={
'host': socket.gethostname(),
'action': change['action'],
'route': change['route']
}
)
r = json.loads(r.text)
print(json.dumps(r, indent=4))
success = r['status'] == 'ok'
# ~ except:
# ~ print("Error uploading, retrying again\n")
actual = deepcopy(new)
new = get_json(ORIGINAL)
return 0
if __name__ == '__main__':
import sys
sys.exit(main(sys.argv))