PageRenderTime 105ms CodeModel.GetById 13ms RepoModel.GetById 0ms app.codeStats 0ms

/syncy.py

https://gitlab.com/shinvdu/SyncY
Python | 928 lines | 890 code | 18 blank | 20 comment | 60 complexity | bfee9d77f5657e926597313907d6d175 MD5 | raw file
  1. #!/usr/bin/env python
  2. # -*- coding:utf-8 -*-
  3. ####################################################################################################
  4. #
  5. # Author: wishinlife
  6. # QQ: 57956720
  7. # QQ Group: 59160264
  8. # E-Mail: wishinlife@qq.com
  9. # Web Home: http://www.syncy.cn
  10. # Update date: 2015-05-26
  11. # VERSION: 2.5.1
  12. # Required packages: kmod-nls-utf8, libopenssl, libcurl, python, python-curl, python-crypto
  13. #
  14. ####################################################################################################
  15. import sys
  16. import os
  17. import stat
  18. import time
  19. import re
  20. import struct
  21. import hashlib
  22. from urllib import urlencode
  23. import threading
  24. import traceback
  25. import json
  26. import random
  27. # import fcntl
  28. # if '/usr/lib/python2.7/site-packages' not in sys.path:
  29. # sys.path.append('/usr/lib/python2.7/site-packages')
  30. import pycurl
  31. import binascii
  32. # import zlib
  33. # import fileinput
  34. try: # require python-crypto
  35. from Crypto.Cipher import ARC4
  36. from Crypto.Cipher import Blowfish
  37. from Crypto.Cipher import AES
  38. except ImportError, ex:
  39. ARC4 = Blowfish = AES = None
  40. # set config_file and pidfile for your config storage path.
  41. if os.name == 'nt':
  42. _CONFIG_FILE = './syncy'
  43. _PIDFILE = './syncy.pid'
  44. _CHARSET = 'GBK' # windows charset
  45. _TMP_DIR = os.environ['TMP'].replace('\\', '/')
  46. else:
  47. _CONFIG_FILE = '/etc/config/syncy'
  48. _PIDFILE = '/var/run/syncy.pid'
  49. _CHARSET = 'UTF-8' # linux charset
  50. _TMP_DIR = '/tmp'
  51. if sys.getdefaultencoding() != _CHARSET:
  52. reload(sys)
  53. sys.setdefaultencoding(_CHARSET)
  54. # Don't modify the following.
  55. _VERSION = '2.5.1'
  56. _DEBUG = False
  57. __author__ = "WishInLife <wishinlife@qq.com>"
  58. if os.name == 'nt':
  59. import win32con
  60. import win32file
  61. import pywintypes
  62. LOCK_SH = 0
  63. LOCK_NB = win32con.LOCKFILE_FAIL_IMMEDIATELY
  64. LOCK_EX = win32con.LOCKFILE_EXCLUSIVE_LOCK
  65. LOCK_UN = 8
  66. __overlapped = pywintypes.OVERLAPPED()
  67. def flock(fd, op):
  68. fh = win32file._get_osfhandle(fd.fileno())
  69. if op == LOCK_UN:
  70. return win32file.UnlockFileEx(fh, 0, 0x0fff0000, __overlapped)
  71. else:
  72. return win32file.LockFileEx(fh, op, 0, 0x0fff0000, __overlapped)
  73. def lockf(fd, op, length=0, start=0, whence=0):
  74. fh = win32file._get_osfhandle(fd.fileno())
  75. fsize = win32file.GetFileSize(fh)
  76. if whence == 1:
  77. start += fd.tell()
  78. elif whence == 2:
  79. start += fsize
  80. if length == 0:
  81. length = fsize
  82. int32 = 2 ** 32
  83. if op == LOCK_UN:
  84. return win32file.UnlockFile(fh, int(start % int32), int(start / int32), int(length % int32), int(length / int32))
  85. else:
  86. return win32file.LockFile(fh, int(start % int32), int(start / int32), int(length % int32), int(length / int32))
  87. else:
  88. from fcntl import LOCK_EX, LOCK_SH, LOCK_NB, LOCK_UN, flock, lockf
  89. LogLock = threading.Lock()
  90. def printlog(msg):
  91. LogLock.acquire()
  92. print(msg)
  93. LogLock.release()
  94. def rename(src, dst):
  95. if os.name == 'nt' and os.path.exists(dst):
  96. os.remove(dst)
  97. os.rename(src, dst)
  98. class SyncY:
  99. synccount = 0
  100. errorcount = 0
  101. failcount = 0
  102. EXLock = threading.Lock()
  103. TaskSemaphore = None
  104. oldSTDERR = None
  105. oldSTDOUT = None
  106. syncydb = None
  107. sydb = None
  108. sydblen = None
  109. syncData = None
  110. basedirlen = None
  111. syncpath = {}
  112. extraslice = None
  113. encryption = None
  114. encryptkey = ''
  115. config = {
  116. 'syncylog' : '',
  117. 'blocksize' : 10,
  118. 'ondup' : 'rename',
  119. 'datacache' : 'on',
  120. 'excludefiles' : '',
  121. 'listnumber' : 100,
  122. 'retrytimes' : 3,
  123. 'retrydelay' : 3,
  124. 'maxsendspeed' : 0,
  125. 'maxrecvspeed' : 0,
  126. 'speedlimitperiod': '0-0',
  127. 'syncperiod' : '0-24',
  128. 'syncinterval' : 3600,
  129. 'tasknumber' : 2,
  130. 'threadnumber' : 2}
  131. syre = {
  132. 'newname': re.compile(r'^(.*)(\.[^.]+)$'),
  133. 'pcspath': re.compile(r'^[\s\.\n].*|.*[/<>\\|\*\?:\"].*|.*[\s\.\n]$')}
  134. syncytoken = {'synctotal': 0}
  135. pcsroot = '/apps/SyncY'
  136. synctask = {}
  137. def __init__(self, argv=sys.argv[1:]):
  138. self.__argv = argv
  139. if len(self.__argv) == 0 or self.__argv[0] in ['compress', 'convert', 'rebuild']:
  140. if os.path.exists(_PIDFILE):
  141. with open(_PIDFILE, 'r') as pidh:
  142. mypid = pidh.read()
  143. try:
  144. os.kill(int(mypid), 0)
  145. except os.error:
  146. pass
  147. else:
  148. print("SyncY is running!")
  149. sys.exit(0)
  150. with open(_PIDFILE, 'w') as pidh:
  151. pidh.write(str(os.getpid()))
  152. if not (os.path.isfile(_CONFIG_FILE)):
  153. sys.stderr.write('%s ERROR: Config file "%s" does not exist.\n' % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), _CONFIG_FILE))
  154. sys.exit(2)
  155. with open(_CONFIG_FILE, 'r') as sycfg:
  156. line = sycfg.readline()
  157. section = ''
  158. while line:
  159. if re.findall(r'^\s*#', line) or re.findall(r'^\s*$', line):
  160. line = sycfg.readline()
  161. continue
  162. line = re.sub(r'#[^\']*$', '', line)
  163. m = re.findall(r'\s*config\s+([^\s]+).*', line)
  164. if m:
  165. section = m[0].strip('\'')
  166. if section == 'syncpath':
  167. SyncY.syncpath[str(len(SyncY.syncpath))] = {}
  168. line = sycfg.readline()
  169. continue
  170. m = re.findall(r'\s*option\s+([^\s]+)\s+\'([^\']*)\'', line)
  171. if m:
  172. if section == 'syncy':
  173. if m[0][0].strip('\'') in ['blocksize', 'listnumber', 'syncinterval', 'threadnumber', 'tasknumber', 'retrytimes', 'retrydelay']:
  174. SyncY.config[m[0][0].strip('\'')] = int(m[0][1])
  175. elif m[0][0].strip('\'') in ['maxsendspeed', 'maxrecvspeed']:
  176. if m[0][1].upper().find('K') > -1:
  177. idx = m[0][1].upper().find('K')
  178. SyncY.config[m[0][0].strip('\'')] = int(m[0][1][0:idx]) * 1024
  179. elif m[0][1].upper().find('M') > -1:
  180. idx = m[0][1].upper().find('M')
  181. SyncY.config[m[0][0].strip('\'')] = int(m[0][1][0:idx]) * 1024 * 1024
  182. else:
  183. SyncY.config[m[0][0].strip('\'')] = int(m[0][1])
  184. else:
  185. SyncY.config[m[0][0].strip('\'')] = m[0][1]
  186. elif section == 'syncytoken':
  187. if m[0][0].strip('\'') in ['expires_in', 'refresh_date', 'compress_date', 'synctotal']:
  188. SyncY.syncytoken[m[0][0].strip('\'')] = int(m[0][1])
  189. else:
  190. SyncY.syncytoken[m[0][0].strip('\'')] = m[0][1]
  191. elif section == 'syncpath':
  192. if m[0][0].strip('\'') == 'remotepath':
  193. if m[0][1].lower().startswith('/apps/syncy'):
  194. SyncY.syncpath[str(len(SyncY.syncpath) - 1)]['remotepath'] = m[0][1][11:]
  195. elif m[0][1].lower().startswith('/我的应用程序/syncy'):
  196. SyncY.syncpath[str(len(SyncY.syncpath) - 1)]['remotepath'] = m[0][1][len('/我的应用程序/syncy'):]
  197. else:
  198. SyncY.syncpath[str(len(SyncY.syncpath) - 1)]['remotepath'] = m[0][1]
  199. else:
  200. SyncY.syncpath[str(len(SyncY.syncpath) - 1)][m[0][0].strip('\'')] = m[0][1].replace('\\', '/')
  201. line = sycfg.readline()
  202. if SyncY.config['syncylog'] != '':
  203. if not os.path.exists(os.path.dirname(SyncY.config['syncylog'])):
  204. os.makedirs(os.path.dirname(SyncY.config['syncylog']))
  205. if os.path.exists(SyncY.config['syncylog']) and os.path.isdir(SyncY.config['syncylog']):
  206. SyncY.config['syncylog'] = self.__catpath(SyncY.config['syncylog'], 'syncy.log')
  207. self.__save_config()
  208. if SyncY.oldSTDERR is None and SyncY.config['syncylog'] != '' and len(self.__argv) != 0 and self.__argv[0] in ['sybind', 'cpbind']:
  209. SyncY.oldSTDERR = sys.stderr
  210. SyncY.oldSTDOUT = sys.stdout
  211. sys.stderr = open(SyncY.config['syncylog'], 'a', 0)
  212. sys.stdout = sys.stderr
  213. if 'refresh_token' not in SyncY.syncytoken or SyncY.syncytoken['refresh_token'] == '' or (len(self.__argv) != 0 and self.__argv[0] in ['sybind', 'cpbind']):
  214. sycurl = SYCurl()
  215. if (('device_code' not in SyncY.syncytoken or SyncY.syncytoken['device_code'] == '') and len(self.__argv) == 0) or (len(self.__argv) != 0 and self.__argv[0] == 'sybind'):
  216. retcode, responses = sycurl.request('https://www.syncy.cn/syserver', {}, {'method': 'bind_device', 'scope': 'basic,netdisk'}, 'POST', SYCurl.Normal)
  217. responses = json.loads(responses)
  218. if retcode != 200:
  219. print('%s ERROR(Errno:%d): Get device code failed: %s.' % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), retcode, responses['error_msg']))
  220. sys.exit(3)
  221. device_code = responses['device_code']
  222. user_code = responses['user_code']
  223. if len(self.__argv) != 0 and self.__argv[0] == 'sybind':
  224. with open(_TMP_DIR + '/syncy.bind', 'w') as sybind:
  225. sybind.write('{"user_code":"%s","device_code":"%s","time":%d}' % (user_code, device_code, int(time.time())))
  226. sys.exit(0)
  227. SyncY.syncytoken['device_code'] = device_code
  228. print('Device binding Guide:')
  229. print(' 1. Open web browser to visit:"https://openapi.baidu.com/device" and input user code to binding your baidu account.')
  230. print(' ')
  231. print(' 2. User code:\033[31m %s\033[0m' % user_code)
  232. print(' (User code valid for 30 minutes.)')
  233. print(' ')
  234. raw_input(' 3. After granting access to the application, come back here and press [Enter] to continue.')
  235. print(' ')
  236. if len(self.__argv) != 0 and self.__argv[0] == 'cpbind':
  237. with open(_TMP_DIR + '/syncy.bind', 'r') as sybind:
  238. bindinfo = sybind.read()
  239. bindinfo = json.loads(bindinfo)
  240. os.remove(_TMP_DIR + '/syncy.bind')
  241. if 'device_code' in bindinfo:
  242. if int(time.time()) - int(bindinfo['time']) >= 1800:
  243. sys.exit(4)
  244. SyncY.syncytoken['device_code'] = bindinfo['device_code']
  245. else:
  246. sys.exit(5)
  247. retcode, responses = sycurl.request('https://www.syncy.cn/syserver', {}, {'method': 'get_device_token', 'code': SyncY.syncytoken['device_code'], 'edition': 'python', 'ver': _VERSION}, 'POST', SYCurl.Normal)
  248. responses = json.loads(responses)
  249. if retcode != 200:
  250. print('%s ERROR(Errno:%d): Get device token failed: %s.' % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), retcode, responses['error_msg']))
  251. sys.exit(6)
  252. SyncY.syncytoken['refresh_token'] = responses['refresh_token']
  253. SyncY.syncytoken['access_token'] = responses['access_token']
  254. SyncY.syncytoken['expires_in'] = int(responses['expires_in'])
  255. SyncY.syncytoken['refresh_date'] = int(time.time())
  256. SyncY.syncytoken['compress_date'] = int(time.time())
  257. self.__save_config()
  258. if len(self.__argv) != 0 and self.__argv[0] == 'cpbind':
  259. sys.exit(0)
  260. print('%s INFO: Get device token success.\n' % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())))
  261. if SyncY.oldSTDERR is None and SyncY.config['syncylog'] != '':
  262. print('%s INFO: Running log output to log file %s.' % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), SyncY.config['syncylog']))
  263. SyncY.oldSTDERR = sys.stderr
  264. SyncY.oldSTDOUT = sys.stdout
  265. sys.stderr = open(SyncY.config['syncylog'], 'a', 0)
  266. sys.stdout = sys.stderr
  267. try:
  268. if SyncY.config['blocksize'] < 1:
  269. SyncY.config['blocksize'] = 10
  270. print('%s WARNING: "blocksize" must great than or equal to 1(M), set to default 10(M).' % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())))
  271. if SyncY.config['ondup'] != 'overwrite' and SyncY.config['ondup'] != 'rename':
  272. SyncY.config['ondup'] = 'rename'
  273. print('%s WARNING: ondup is invalid, set to default(overwrite).' % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())))
  274. if SyncY.config['datacache'] != 'on' and SyncY.config['datacache'] != 'off':
  275. SyncY.config['datacache'] = 'on'
  276. print('%s WARNING: "datacache" is invalid, set to default(on).' % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())))
  277. if SyncY.config['retrytimes'] < 0:
  278. SyncY.config['retrytimes'] = 3
  279. print('%s WARNING: "retrytimes" is invalid, set to default(3 times).' % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())))
  280. if SyncY.config['retrydelay'] < 0:
  281. SyncY.config['retrydelay'] = 3
  282. print('%s WARNING: "retrydelay" is invalid, set to default(3 second).' % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())))
  283. if SyncY.config['listnumber'] < 1:
  284. SyncY.config['listnumber'] = 100
  285. print('%s WARNING: "listnumber" must great than or equal to 1, set to default 100.' % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())))
  286. if SyncY.config['syncinterval'] < 0:
  287. SyncY.config['syncinterval'] = 3600
  288. print('%s WARNING: "syncinterval" must great than or equal to 1, set to default 3600.' % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())))
  289. if SyncY.config['maxsendspeed'] < 0:
  290. SyncY.config['maxsendspeed'] = 0
  291. print('%s WARNING: "maxsendspeed" must great than or equal to 0, set to default 0.' % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())))
  292. if SyncY.config['maxrecvspeed'] < 0:
  293. SyncY.config['maxrecvspeed'] = 0
  294. print('%s WARNING: "maxrecvspeed" must great than or equal to 0, set to default 100.' % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())))
  295. if SyncY.config['threadnumber'] < 1:
  296. SyncY.config['threadnumber'] = 2
  297. print('%s WARNING: "threadnumber" must great than or equal to 1, set to default 2.' % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())))
  298. if SyncY.config['tasknumber'] < 1:
  299. SyncY.config['tasknumber'] = 2
  300. print('%s WARNING: "tasknumber" must great than or equal to 1, set to default 2.' % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())))
  301. starthour, endhour = SyncY.config['speedlimitperiod'].split('-', 1)
  302. if starthour == '' or endhour == '' or int(starthour) < 0 or int(starthour) > 23 or int(endhour) < 0 or int(endhour) > 24:
  303. print('%s WARNING: "speedlimitperiod" is invalid, set to default(0-0), no limit.' % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())))
  304. SyncY.config['speedlimitperiod'] = '0-0'
  305. starthour, endhour = SyncY.config['syncperiod'].split('-', 1)
  306. if starthour == '' or endhour == '' or int(starthour) < 0 or int(starthour) > 23 or int(endhour) < 0 or int(endhour) > 24 or endhour == starthour:
  307. print('%s WARNING: "syncperiod" is invalid, set to default(0-24).' % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())))
  308. SyncY.config['syncperiod'] = '0-24'
  309. except Exception, e:
  310. print('%s ERROR: initialize parameters failed. %s\n%s' % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), e, traceback.format_exc()))
  311. sys.exit(7)
  312. self._excludefiles = SyncY.config['excludefiles'].replace('\\', '/').replace('.', '\.').replace('*', '.*').replace('?', '.?').split(';')
  313. for i in xrange(len(self._excludefiles)):
  314. self._excludefiles[i] = re.compile(eval('r"^' + self._excludefiles[i] + '$"'))
  315. self._excludefiles.append(re.compile(r'^.*\.syy$'))
  316. if (SyncY.syncytoken['refresh_date'] + SyncY.syncytoken['expires_in'] - 864000) < int(time.time()):
  317. self.__check_expires()
  318. SyncY.TaskSemaphore = threading.Semaphore(SyncY.config['tasknumber'])
  319. size = 32768
  320. while True:
  321. try:
  322. threading.stack_size(size)
  323. threadtest = ThreadTest()
  324. threadtest.start()
  325. break
  326. except threading.ThreadError:
  327. threading.stack_size(0)
  328. break
  329. except RuntimeError:
  330. threading.stack_size(0)
  331. break
  332. except ValueError:
  333. if size < 512 * 1024:
  334. size *= 2
  335. else:
  336. threading.stack_size(0)
  337. break
  338. def __del__(self):
  339. if self.__class__.oldSTDERR is not None:
  340. sys.stderr.flush()
  341. sys.stderr.close()
  342. sys.stderr = self.__class__.oldSTDERR
  343. sys.stdout = self.__class__.oldSTDOUT
  344. if os.path.exists(_PIDFILE):
  345. with open(_PIDFILE, 'r') as pidh:
  346. lckpid = pidh.read()
  347. if os.getpid() == int(lckpid):
  348. os.remove(_PIDFILE)
  349. @staticmethod
  350. def synccount_increase():
  351. SyncY.EXLock.acquire()
  352. SyncY.synccount += 1
  353. SyncY.EXLock.release()
  354. @staticmethod
  355. def errorcount_increase():
  356. SyncY.EXLock.acquire()
  357. SyncY.errorcount += 1
  358. SyncY.EXLock.release()
  359. @staticmethod
  360. def failcount_increase():
  361. SyncY.EXLock.acquire()
  362. SyncY.failcount += 1
  363. SyncY.EXLock.release()
  364. @staticmethod
  365. def reset_counter():
  366. SyncY.EXLock.acquire()
  367. SyncY.synccount = 0
  368. SyncY.failcount = 0
  369. SyncY.errorcount = 0
  370. SyncY.EXLock.release()
  371. @staticmethod
  372. def __init_syncdata():
  373. SyncY.syncData = {}
  374. if os.path.exists(SyncY.syncydb):
  375. with open(SyncY.syncydb, 'rb') as sydb:
  376. flock(sydb, LOCK_SH)
  377. sydb.seek(64)
  378. datarec = sydb.read(64)
  379. while datarec:
  380. SyncY.syncData[datarec[0:16]] = datarec[16:]
  381. datarec = sydb.read(64)
  382. flock(sydb, LOCK_UN)
  383. def __check_expires(self):
  384. sycurl = SYCurl()
  385. retcode, responses = sycurl.request('https://openapi.baidu.com/rest/2.0/passport/users/getLoggedInUser', {}, {'access_token': SyncY.syncytoken['access_token']}, 'POST', SYCurl.Normal)
  386. responses = json.loads(responses)
  387. if 'uid' in responses:
  388. retcode, responses = sycurl.request('https://www.syncy.cn/syserver', {}, {'method': 'get_last_version', 'edition': 'python', 'ver': _VERSION, 'uid': responses['uid'], 'code': SyncY.syncytoken['device_code']}, 'POST', SYCurl.Normal)
  389. if retcode == 200 and responses.find('#') > -1:
  390. (lastver, smessage) = responses.strip('\n').split('#', 1)
  391. if lastver > _VERSION:
  392. printlog('%s WARNING: %s' % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), smessage.encode(_CHARSET)))
  393. if (SyncY.syncytoken['refresh_date'] + SyncY.syncytoken['expires_in'] - 864000) > int(time.time()):
  394. return
  395. retcode, responses = sycurl.request('https://www.syncy.cn/syserver', {}, {'method': 'refresh_access_token', 'refresh_token': SyncY.syncytoken['refresh_token'], 'code': SyncY.syncytoken['device_code']}, 'POST', SYCurl.Normal)
  396. responses = json.loads(responses)
  397. if retcode != 200:
  398. printlog('%s ERROR(Errno:%d): Refresh access token failed: %s.' % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), retcode, responses['error_msg']))
  399. return 1
  400. SyncY.syncytoken['refresh_token'] = responses['refresh_token']
  401. SyncY.syncytoken['access_token'] = responses['access_token']
  402. SyncY.syncytoken['expires_in'] = int(responses['expires_in'])
  403. SyncY.syncytoken['refresh_date'] = int(time.time())
  404. self.__save_config()
  405. printlog('%s INFO: Refresh access token success.' % time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()))
  406. return 0
  407. @staticmethod
  408. def __save_config():
  409. with open('%s.tmp' % _CONFIG_FILE, 'w') as sycfg:
  410. sycfg.write("\nconfig syncy\n")
  411. for key, value in SyncY.config.items():
  412. sycfg.write("\toption %s '%s'\n" % (key, str(value)))
  413. sycfg.write("\nconfig syncytoken\n")
  414. for key, value in SyncY.syncytoken.items():
  415. sycfg.write("\toption %s '%s'\n" % (key, str(value)))
  416. for i in range(len(SyncY.syncpath)):
  417. sycfg.write("\nconfig syncpath\n")
  418. for key, value in SyncY.syncpath[str(i)].items():
  419. sycfg.write("\toption %s '%s'\n" % (key, str(value)))
  420. sycfg.flush()
  421. os.fsync(sycfg.fileno())
  422. if os.path.exists('%s.tmp' % _CONFIG_FILE):
  423. pmeta = os.stat(_CONFIG_FILE)
  424. rename('%s.tmp' % _CONFIG_FILE, _CONFIG_FILE)
  425. if os.name == 'posix':
  426. os.lchown(_CONFIG_FILE, pmeta.st_uid, pmeta.st_gid)
  427. os.chmod(_CONFIG_FILE, pmeta.st_mode)
  428. @staticmethod
  429. def __catpath(*names):
  430. fullpath = '/'.join(names)
  431. fullpath = re.sub(r'/+', '/', fullpath)
  432. fullpath = re.sub(r'/$', '', fullpath)
  433. return fullpath
  434. @staticmethod
  435. def __get_newname(oldname):
  436. nowtime = str(time.strftime("%Y%m%d%H%M%S", time.localtime()))
  437. m = SyncY.syre['newname'].findall(oldname)
  438. if m:
  439. newname = m[0][0] + '_old_' + nowtime + m[0][1]
  440. else:
  441. newname = oldname + '_old_' + nowtime
  442. return newname
  443. @staticmethod
  444. def __check_pcspath(pcsdirname, pcsfilename):
  445. if len(pcsdirname) + len(pcsfilename) + 1 >= 1000:
  446. printlog('%s ERROR: Length of PCS path(%s/%s) must less than 1000, skip upload.' % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), pcsdirname, pcsfilename))
  447. return 1
  448. if SyncY.syre['pcspath'].findall(pcsfilename):
  449. printlog('%s ERROR: PCS path(%s/%s) is invalid, please check whether special characters exists in the path, skip upload the file.' % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), pcsdirname, pcsfilename))
  450. return 1
  451. return 0
  452. @staticmethod
  453. def __get_pcs_quota():
  454. sycurl = SYCurl()
  455. retcode, responses = sycurl.request('https://pcs.baidu.com/rest/2.0/pcs/quota', {'method': 'info', 'access_token': SyncY.syncytoken['access_token']}, '', 'GET', SYCurl.Normal)
  456. responses = json.loads(responses)
  457. if retcode != 200:
  458. printlog('%s ERROR(Errno:%d): Get pcs quota failed: %s.' % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), retcode, responses['error_msg']))
  459. return 1
  460. printlog('%s INFO: PCS quota is %dG,used %dG.' % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), responses['quota'] / 1024 / 1024 / 1024, responses['used'] / 1024 / 1024 / 1024))
  461. return 0
  462. @staticmethod
  463. def __get_pcs_filelist(pcspath, startindex, endindex):
  464. if _DEBUG:
  465. printlog('%s Info(%s): Start get pcs file list(%d-%d) of "%s".' % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), threading.currentThread().name, startindex, endindex, pcspath))
  466. sycurl = SYCurl()
  467. retcode, responses = sycurl.request('https://pcs.baidu.com/rest/2.0/pcs/file', {'method': 'list', 'access_token': SyncY.syncytoken['access_token'], 'path': pcspath, 'limit': '%d-%d' % (startindex, endindex), 'by': 'name', 'order': 'asc'}, '', 'GET', SYCurl.Normal)
  468. try:
  469. responses = json.loads(responses)
  470. if retcode != 200:
  471. if responses['error_code'] == 31066:
  472. return 31066, []
  473. else:
  474. printlog('%s ERROR(Errno:%d): Get PCS file list of "%s" failed: %s.' % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), retcode, pcspath, responses['error_msg']))
  475. return 1, []
  476. return 0, responses['list']
  477. except Exception, e:
  478. printlog('%s ERROR: Get PCS file list of "%s" failed. return code: %d, response body: %s.\n%s\n%s' % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), pcspath, retcode, str(responses), e, traceback.format_exc()))
  479. return 1, []
  480. finally:
  481. del responses
  482. if _DEBUG:
  483. printlog('%s Info(%s): Complete get pcs file list(%d-%d) of "%s".' % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), threading.currentThread().name, startindex, endindex, pcspath))
  484. @staticmethod
  485. def __check_create_pcsdir(pcspath):
  486. sycurl = SYCurl()
  487. retcode, responses = sycurl.request('https://pcs.baidu.com/rest/2.0/pcs/file', {'method': 'meta', 'access_token': SyncY.syncytoken['access_token'], 'path': pcspath}, '', 'GET', SYCurl.Normal)
  488. try:
  489. responses = json.loads(responses)
  490. if retcode == 200 and responses['list'][0]['isdir'] == 1:
  491. return 0
  492. elif (retcode != 200 and responses['error_code'] == 31066) or (retcode == 200 and responses['list'][0]['isdir'] == 0):
  493. retcode, responses = sycurl.request('https://pcs.baidu.com/rest/2.0/pcs/file', {'method': 'mkdir', 'access_token': SyncY.syncytoken['access_token'], 'path': pcspath}, '', 'POST', SYCurl.Normal)
  494. responses = json.loads(responses)
  495. if retcode == 200 and responses['path'].encode(_CHARSET) == pcspath:
  496. return 0
  497. printlog('%s ERROR(Errno:%d): Create PCS directory "%s" failed: %s.' % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), retcode, pcspath, responses['error_msg']))
  498. return 1
  499. except Exception, e:
  500. printlog('%s ERROR: Create PCS directory "%s" failed. return code: %d, response body: %s.\n%s\n%s' % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), pcspath, retcode, str(responses), e, traceback.format_exc()))
  501. return 1
  502. def __rm_localfile(self, delpath, slient=False):
  503. try:
  504. if os.path.isfile(delpath):
  505. os.remove(delpath)
  506. if not slient:
  507. printlog('%s INFO: Delete local file "%s" completed.' % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), delpath))
  508. elif os.path.isdir(delpath):
  509. fnlist = os.listdir(delpath)
  510. for i in xrange(len(fnlist)):
  511. self.__rm_localfile('%s/%s' % (delpath, fnlist[i]), slient)
  512. os.rmdir(delpath)
  513. if not slient:
  514. printlog('%s INFO: Delete local directory "%s" completed.' % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), delpath))
  515. except Exception, e:
  516. if not slient:
  517. printlog('%s ERROR: Delete local file "%s" failed. %s\n%s' % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), delpath, e, traceback.format_exc()))
  518. return 1
  519. return 0
  520. @staticmethod
  521. def __rm_pcsfile(pcspath, slient=False):
  522. sycurl = SYCurl()
  523. retcode, responses = sycurl.request('https://pcs.baidu.com/rest/2.0/pcs/file', {'method': 'delete', 'access_token': SyncY.syncytoken['access_token'], 'path': pcspath}, '', 'POST', SYCurl.Normal)
  524. responses = json.loads(responses)
  525. if retcode != 200:
  526. if not slient:
  527. printlog('%s ERROR(Errno:%d): Delete remote file or directory "%s" failed: %s.' % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), retcode, pcspath, responses['error_msg']))
  528. return 1
  529. if not slient:
  530. printlog('%s INFO: Delete remote file or directory "%s" completed.' % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), pcspath))
  531. return 0
  532. @staticmethod
  533. def __mv_pcsfile(oldpcspath, newpcspath, slient=False):
  534. sycurl = SYCurl()
  535. retcode, responses = sycurl.request('https://pcs.baidu.com/rest/2.0/pcs/file', {'method': 'move', 'access_token': SyncY.syncytoken['access_token'], 'from': oldpcspath, 'to': newpcspath}, '', 'POST', SYCurl.Normal)
  536. responses = json.loads(responses)
  537. if retcode != 200:
  538. if not slient:
  539. printlog('%s ERROR(Errno:%d): Move remote file or directory "%s" to "%s" failed: %s.' % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), retcode, oldpcspath, newpcspath, responses['error_msg']))
  540. return 1
  541. if not slient:
  542. printlog('%s INFO: Move remote file or directory "%s" to "%s" completed.' % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), oldpcspath, newpcspath))
  543. return 0
  544. @staticmethod
  545. def __cp_pcsfile(srcpcspath, destpcspath):
  546. sycurl = SYCurl()
  547. retcode, responses = sycurl.request('https://pcs.baidu.com/rest/2.0/pcs/file', {'method': 'copy', 'access_token': SyncY.syncytoken['access_token'], 'from': srcpcspath, 'to': destpcspath}, '', 'POST', SYCurl.Normal)
  548. responses = json.loads(responses)
  549. if retcode != 200:
  550. printlog('%s ERROR(Errno:%d): Copy remote file or directory "%s" to "%s" failed: %s.' % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), retcode, srcpcspath, destpcspath, responses['error_msg']))
  551. return 1
  552. printlog('%s INFO: Copy remote file or directory "%s" to "%s" completed.' % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), srcpcspath, destpcspath))
  553. return 0
  554. @staticmethod
  555. def __get_pcs_filemeta(pcspath):
  556. sycurl = SYCurl()
  557. retcode, responses = sycurl.request('https://pcs.baidu.com/rest/2.0/pcs/file', {'method': 'meta', 'access_token': SyncY.syncytoken['access_token'], 'path': pcspath}, '', 'GET', SYCurl.Normal)
  558. responses = json.loads(responses)
  559. if retcode != 200:
  560. printlog('%s ERROR(Errno:%d): Get file\'s meta failed: %s, %s.' % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), retcode, pcspath, responses['error_msg']))
  561. return 1, {}
  562. return 0, responses['list'][0]
  563. @staticmethod
  564. def __upload_file_nosync(filepath, pcspath):
  565. sycurl = SYCurl()
  566. retcode, responses = sycurl.request('https://c.pcs.baidu.com/rest/2.0/pcs/file', {'method': 'upload', 'access_token': SyncY.syncytoken['access_token'], 'path': pcspath, 'ondup': 'newcopy'}, '0-%d' % (os.stat(filepath).st_size - 1), 'POST', SYCurl.Upload, filepath)
  567. responses = json.loads(responses)
  568. if retcode != 200:
  569. printlog('%s ERROR(Errno:%d): Upload file to pcs failed: %s, %s.' % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), retcode, filepath, responses['error_msg']))
  570. return 1
  571. printlog('%s INFO: Upload file "%s" completed.' % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), filepath))
  572. return 0
  573. def __compress_data(self, pathname, sydbnew, sydb=None, sydblen=0):
  574. fnlist = os.listdir(pathname)
  575. fnlist.sort()
  576. for fnname in fnlist:
  577. if fnname[0:1] == '.':
  578. continue
  579. fullpath = '%s/%s' % (pathname, fnname)
  580. if os.path.isdir(fullpath):
  581. if SyncY.config['datacache'] == 'on':
  582. self.__compress_data(fullpath, sydbnew)
  583. else:
  584. self.__compress_data(fullpath, sydbnew, sydb, sydblen)
  585. elif os.path.isfile(fullpath):
  586. fnmd5 = hashlib.md5(fullpath[SyncY.basedirlen:]).digest()
  587. fnstat = os.stat(fullpath)
  588. fmate = struct.pack('>qq', int(fnstat.st_mtime), fnstat.st_size)
  589. if SyncY.config['datacache'] == 'on':
  590. if fnmd5 in SyncY.syncData and SyncY.syncData[fnmd5][0:16] == fmate:
  591. sydbnew.write('%s%s' % (fnmd5, SyncY.syncData[fnmd5]))
  592. del SyncY.syncData[fnmd5]
  593. else:
  594. if sydb.tell() == sydblen:
  595. sydb.seek(64)
  596. datarec = sydb.read(64)
  597. readlen = 64
  598. while datarec and readlen <= sydblen - 64:
  599. if datarec[0:32] == '%s%s' % (fnmd5, fmate):
  600. sydbnew.write(datarec)
  601. break
  602. if readlen == sydblen - 64:
  603. break
  604. if sydb.tell() == sydblen:
  605. sydb.seek(64)
  606. datarec = sydb.read(64)
  607. readlen += 64
  608. return 0
  609. def __start_compress(self, pathname=''):
  610. if pathname == '':
  611. mpath = []
  612. for i in range(len(SyncY.syncpath)):
  613. if SyncY.syncpath[str(i)]['synctype'].lower() not in ['4', 's', 'sync']:
  614. mpath.append(SyncY.syncpath[str(i)]['localpath'])
  615. printlog('%s INFO: Start compress sync data.' % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())))
  616. else:
  617. mpath = [pathname]
  618. for ipath in mpath:
  619. if ipath == '':
  620. continue
  621. SyncY.basedirlen = len(ipath)
  622. SyncY.syncydb = '%s/.syncy.info.db' % ipath
  623. newdbfile = '%s/.syncy.info.dbtmp' % ipath
  624. if os.path.exists(SyncY.syncydb):
  625. if os.path.exists(newdbfile):
  626. os.remove(newdbfile)
  627. self.__check_upgrade_syncdata(newdbfile)
  628. with open(newdbfile, 'ab') as sydbnew:
  629. if SyncY.config['datacache'] == 'on':
  630. self.__init_syncdata()
  631. self.__compress_data(ipath, sydbnew)
  632. SyncY.syncData = None
  633. else:
  634. sydblen = os.stat(SyncY.syncydb).st_size
  635. with open(SyncY.syncydb, 'rb') as sydb:
  636. self.__compress_data(ipath, sydbnew, sydb, sydblen)
  637. sydbnew.flush()
  638. os.fsync(sydbnew.fileno())
  639. rename(newdbfile, SyncY.syncydb)
  640. if pathname == '':
  641. SyncY.syncytoken['compress_date'] = int(time.time())
  642. SyncY.syncytoken['synctotal'] = 0
  643. self.__save_config()
  644. printlog('%s INFO: Sync data compress completed.' % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())))
  645. def __check_excludefiles(self, filepath):
  646. for reexf in self._excludefiles:
  647. if reexf.findall(filepath):
  648. return 1
  649. return 0
  650. @staticmethod
  651. def __check_syncstatus(fmd5, fmate, rmate, rmd5):
  652. if rmd5 != '*':
  653. rmd5 = rmd5.decode('hex')
  654. if SyncY.config['datacache'] == 'on':
  655. if fmd5 not in SyncY.syncData:
  656. return 0
  657. if rmd5 == '*' and rmate == '*' and SyncY.syncData[fmd5][0:16] == fmate:
  658. return 1
  659. elif fmate == '*' and SyncY.syncData[fmd5][16:] == rmate + rmd5:
  660. return 1
  661. elif SyncY.syncData[fmd5] == fmate + rmate + rmd5:
  662. return 1
  663. else:
  664. if SyncY.sydb.tell() == SyncY.sydblen:
  665. SyncY.sydb.seek(64)
  666. datarec = SyncY.sydb.read(64)
  667. readlen = 64
  668. while datarec and readlen <= SyncY.sydblen - 64:
  669. if rmd5 == '*' and rmate == '*' and datarec[0:32] == fmd5 + fmate:
  670. return 1
  671. elif fmate == '*' and datarec[16:] == rmate + rmd5:
  672. return 1
  673. elif datarec == fmd5 + fmate + rmate + rmd5:
  674. return 1
  675. if readlen == SyncY.sydblen - 64:
  676. break
  677. if SyncY.sydb.tell() == SyncY.sydblen:
  678. SyncY.sydb.seek(64)
  679. datarec = SyncY.sydb.read(64)
  680. readlen += 64
  681. return 0
  682. def __syncy_upload(self, ldir, rdir):
  683. fnlist = os.listdir(ldir)
  684. fnlist.sort()
  685. for fi in xrange(len(fnlist)):
  686. lfullpath = '%s/%s' % (ldir, fnlist[fi])
  687. fmtime = 0
  688. fsize = 0
  689. try:
  690. if fnlist[fi][0:1] == '.' or self.__check_excludefiles(lfullpath) == 1 or self.__check_pcspath(rdir, fnlist[fi]) == 1:
  691. continue
  692. if _DEBUG:
  693. printlog('%s Info(%s): Start upload "%s".' % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), threading.currentThread().name, lfullpath))
  694. rfullpath = '%s/%s' % (rdir, fnlist[fi])
  695. if os.path.isdir(lfullpath):
  696. self.__syncy_upload(lfullpath, rfullpath)
  697. else:
  698. fmeta = os.stat(lfullpath)
  699. fmtime = int(fmeta.st_mtime)
  700. fsize = fmeta.st_size
  701. fnmd5 = hashlib.md5(lfullpath[SyncY.basedirlen:]).digest()
  702. if self.__check_syncstatus(fnmd5, struct.pack('>qq', fmtime, fsize), '*', '*') == 0:
  703. if SyncY.config['ondup'] == 'rename':
  704. ondup = 'newcopy'
  705. else:
  706. ondup = 'overwrite'
  707. if SyncY.TaskSemaphore.acquire():
  708. synctask = SYTask(SYTask.Upload, lfullpath, int(fmeta.st_mtime), fmeta.st_size, fnmd5, rfullpath, 0, 0, '', ondup)
  709. synctask.start()
  710. else:
  711. continue
  712. except struct.error, e:
  713. printlog('%s ERROR: Struct.pack upload file mate(mtime:%d,size:%d) of "%s" error: %s\n%s.' % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), fmtime, fsize, lfullpath, e, traceback.format_exc()))
  714. self.errorcount_increase()
  715. return 1
  716. except Exception, e:
  717. printlog('%s ERROR: Upload file "%s" failed. %s\n%s' % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), lfullpath, e, traceback.format_exc()))
  718. self.errorcount_increase()
  719. return 1
  720. return 0
  721. def __syncy_uploadplus(self, ldir, rdir):
  722. startidx = 0
  723. retcode, rfnlist = self.__get_pcs_filelist(rdir, startidx, SyncY.config['listnumber'])
  724. if retcode != 0 and retcode != 31066:
  725. self.errorcount_increase()
  726. return 1
  727. lfnlist = os.listdir(ldir)
  728. lfnlist.sort()
  729. while retcode == 0:
  730. for i in xrange(len(rfnlist)):
  731. rfullpath = rfnlist[i]['path'].encode(_CHARSET)
  732. fnname = os.path.basename(rfullpath)
  733. lfullpath = '%s/%s' % (ldir, fnname)
  734. try:
  735. if self.__check_excludefiles(lfullpath) == 1:
  736. continue
  737. if os.path.exists(lfullpath):
  738. for idx in xrange(len(lfnlist)):
  739. if lfnlist[idx] == fnname:
  740. del lfnlist[idx]
  741. break
  742. else:
  743. continue
  744. if _DEBUG:
  745. printlog('%s Info(%s): Start upload+ "%s".' % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), threading.currentThread().name, lfullpath))
  746. if (rfnlist[i]['isdir'] == 1 and os.path.isfile(lfullpath)) or (rfnlist[i]['isdir'] == 0 and os.path.isdir(lfullpath)):
  747. if SyncY.config['ondup'] == 'rename':
  748. fnnamenew = '%s/%s' % (rdir, self.__get_newname(fnname))
  749. if len(fnnamenew) >= 1000:
  750. printlog('%s ERROR: Rename failed, the length of PCS path "%s" must less than 1000, skip upload "%s".' % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), fnnamenew, lfullpath))
  751. self.failcount_increase()
  752. continue
  753. if self.__mv_pcsfile(rfullpath, fnnamenew, True) == 1:
  754. printlog('%s ERROR: Rename "%s" failed, skip upload "%s".' % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), rfullpath, lfullpath))
  755. self.errorcount_increase()
  756. continue
  757. else:
  758. self.__rm_pcsfile(rfullpath, True)
  759. if os.path.isdir(lfullpath):
  760. self.__syncy_uploadplus(lfullpath, rfullpath)
  761. continue
  762. else:
  763. fmeta = os.stat(lfullpath)
  764. fnmd5 = hashlib.md5(lfullpath[SyncY.basedirlen:]).digest()
  765. if SyncY.TaskSemaphore.acquire():
  766. synctask = SYTask(SYTask.Upload, lfullpath, int(fmeta.st_mtime), fmeta.st_size, fnmd5, rfullpath, 0, 0, '', 'overwrite')
  767. synctask.start()
  768. elif rfnlist[i]['isdir'] == 1:
  769. self.__syncy_uploadplus(lfullpath, rfullpath)
  770. continue
  771. else:
  772. fmeta = os.stat(lfullpath)
  773. fnmd5 = hashlib.md5(lfullpath[SyncY.basedirlen:]).digest()
  774. if self.__check_syncstatus(fnmd5, struct.pack('>qq', int(fmeta.st_mtime), fmeta.st_size), struct.pack('>qq', rfnlist[i]['mtime'], rfnlist[i]['size']), rfnlist[i]['md5']) == 1:
  775. continue
  776. if SyncY.config['ondup'] == 'rename':
  777. fnnamenew = '%s/%s' % (rdir, self.__get_newname(fnname))
  778. if len(fnnamenew) >= 1000:
  779. printlog('%s ERROR: Rename failed, the length of PCS path "%s" must less than 1000, skip upload "%s".' % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), fnnamenew, lfullpath))
  780. self.failcount_increase()
  781. continue
  782. if self.__mv_pcsfile(rfullpath, fnnamenew, True) == 1:
  783. printlog('%s ERROR: Rename "%s" failed, skip upload "%s".' % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), rfullpath, lfullpath))
  784. self.failcount_increase()
  785. continue
  786. else:
  787. self.__rm_pcsfile(rfullpath, True)
  788. if SyncY.TaskSemaphore.acquire():
  789. synctask = SYTask(SYTask.Upload, lfullpath, int(fmeta.st_mtime), fmeta.st_size, fnmd5, rfullpath, 0, 0, '', 'overwrite')
  790. synctask.start()
  791. except struct.error, e:
  792. printlog('%s ERROR: Struct.pack file mate of "%s" error: %s\n%s.' % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), lfullpath, e, traceback.format_exc()))
  793. self.errorcount_increase()
  794. return 1
  795. except Exception, e:
  796. printlog('%s ERROR: Upload file "%s" failed. %s\n%s' % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), lfullpath, e, traceback.format_exc()))
  797. self.errorcount_increase()
  798. return 1
  799. if len(rfnlist) < SyncY.config['listnumber']:
  800. break
  801. startidx += SyncY.config['listnumber']
  802. retcode, rfnlist = self.__get_pcs_filelist(rdir, startidx, startidx + SyncY.config['listnumber'])
  803. if retcode != 0:
  804. self.errorcount_increase()
  805. return 1
  806. for idx in xrange(len(lfnlist)):
  807. lfullpath = '%s/%s' % (ldir, lfnlist[idx])
  808. try:
  809. if lfnlist[idx][0:1] == '.' or self.__check_excludefiles(lfullpath) == 1 or self.__check_pcspath(rdir, lfnlist[idx]) == 1:
  810. continue
  811. rfullpath = '%s/%s' % (rdir, lfnlist[idx])
  812. if os.path.isdir(lfullpath):
  813. self.__syncy_uploadplus(lfullpath, rfullpath)
  814. elif os.path.isfile(lfullpath):
  815. fmeta = os.stat(lfullpath)
  816. fnmd5 = hashlib.md5(lfullpath[SyncY.basedirlen:]).digest()
  817. if SyncY.TaskSemaphore.acquire():
  818. synctask = SYTask(SYTask.Upload, lfullpath, int(fmeta.st_mtime), fmeta.st_size, fnmd5, rfullpath, 0, 0, '', 'overwrite')
  819. synctask.start()
  820. except struct.error, e:
  821. printlog('%s ERROR: Struct.pack file mate of "%s" error: %s\n%s.' % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), lfullpath, e, traceback.format_exc()))
  822. self.errorcount_increase()
  823. return 1
  824. except Exception, e:
  825. printlog('%s ERROR: Upload file "%s" failed. %s\n%s' % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), lfullpath, e, traceback.format_exc()))
  826. self.errorcount_increase()
  827. return 1
  828. return 0
  829. def __syncy_download(self, ldir, rdir):
  830. startidx = 0
  831. retcode, rfnlist = self.__get_pcs_filelist(rdir, startidx, SyncY.config['listnumber'])
  832. if retcode != 0:
  833. self.errorcount_increase()
  834. return 1
  835. while retcode == 0:
  836. for i in xrange(len(rfnlist)):
  837. rfullpath = rfnlist[i]['path'].encode(_CHARSET)
  838. fnname = os.path.basename(rfullpath)
  839. if self.__check_excludefiles(rfullpath) == 1:
  840. continue
  841. lfullpath = '%s/%s' % (ldir, fnname)
  842. try:
  843. if _DEBUG:
  844. printlog('%s Info(%s): Start download "%s".' % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), threading.currentThread().name, rfullpath))
  845. if rfnlist[i]['isdir'] == 1:
  846. if os.path.exists(lfullpath) and os.path.isfile(lfullpath):
  847. if SyncY.config['ondup'] == 'rename':
  848. fnnamenew = '%s/%s' % (ldir, self.__get_newname(fnname))
  849. rename(lfullpath, fnnamenew)
  850. else:
  851. if self.__rm_localfile(lfullpath, True) == 1:
  852. printlog('%s ERROR: Delete local file "%s" failed, skip download "%s".' % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), lfullpath, rfullpath))
  853. self.errorcount_increase()
  854. continue
  855. if not (os.path.exists(lfullpath)):
  856. os.mkdir(lfullpath)
  857. if os.name == 'posix':
  858. pmeta = os.stat(ldir)
  859. os.lchown(lfullpath, pmeta.st_uid, pmeta.st_gid)
  860. os.chmod(lfullpath, pmeta.st_mode)
  861. self.__syncy_download(lfullpath, rfullpath)
  862. else:
  863. fnmd5 = hashlib.md5(lfullpath[SyncY.basedirlen:]).digest()
  864. if not (os.path.exists(lfullpath + '.db.syy')):
  865. if self.__check_syncstatus(fnmd5, '*', struct.pack('>qq', rfnlist[i]['mtime'], rfnlist[i]['size']), rfnlist[i]['md5']) == 1:
  866. continue
  867. if os.path.exists(lfullpath) and SyncY.config['ondup'] == 'rename':
  868. fnnamenew = '%s/%s' % (ldir, self.__get_newname(fnname))
  869. rename(lfullpath, fnnamenew)
  870. elif os.path.exists(lfullpath):
  871. if self.__rm_localfile(lfullpath, True) == 1:
  872. printlog('%s ERROR: Delete local file "%s" failed, skip download "%s".' % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), lfullpath, rfullpath))
  873. self.failcount_increase()
  874. continue
  875. if SyncY.TaskSemaphore.acquire():
  876. synctask = SYTask(SYTask.Download, lfullpath, 0, 0, fnmd5, rfullpath, rfnlist[i]['mtime'], rfnlist[i]['size'], rfnlist[i]['md5'], 'overwrite')
  877. synctask.start()
  878. except struct.error, e:
  879. printlog('%s ERROR: Struct.pack file mate of "%s" error: %s\n%s.' % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), lfullpath, e, traceback.format_exc()))
  880. self.errorcount_increase()
  881. return 1
  882. except Exception, e:
  883. printlog('%s ERROR: Download file "%s" failed. %s\n%s' % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), lfullpath, e, traceback.format_exc()))
  884. self.errorcount_increase()