1
2
3
4
5
6
7
8
9
10
11
12
13
14
15 """
16 B{PYBAG} - Crossplatform files synchronization and backup portable tools.
17
18 """
19
20
21
22
23
24
25
26
27 import shutil
28 import sys
29 import time
30 import getopt
31 import atexit
32 import ConfigParser
33 import gzip
34 import binascii
35 import traceback
36 import codecs
37 import os
38 import re
39
40
41 import cfg
42 from const import *
43 from iof import log, prt0, prt1, prt2, prt3, prt4, prt5, dbg, z_u, prt, log_clear,\
44 init_encoding
45 from help import __header__, _usage, __about__, __license__
46 from version import __version__, __author__, __copyright__
47 import termcolors
48
49
50
51 _module = sys.modules[__name__]
52
53 climenu = u"""
54 Enter action (h - for help):
55
56 """
57
58 climenuhelp = u"""
59 <g>Menu help:</g>
60 <g>-------------</g>
61 <y>d DIR DIR</y> - change rules for autoconflict resolution
62 (after change direction comparison will be repeated,
63 and menu displayed again). DIR is next:
64 [<c>bag</c>, <c>origin</c>, <c>no</c>, <c>newer</c>, <c>older</c>, <c>bagall</c>, <c>originall</c>,
65 <c>bagcopy</c>, <c>origincopy</c>].
66 <y>m FLAG</y> - change emulation level (<c>0</c> or <c>1</c>).
67 <y>t FLAG</y> - change debug level (<c>0</c> or <c>1</c>).
68 <y>v FLAG</y> - change verbosity level (<c>-1</c> to <c>5</c>).
69 <y>g OUTFILTER</y> - change display filter (if no OUTFILTER - use default).
70 <c>b</c> - bag, <c>o</c> - origin, <c>n</c> - new, <c>h</c> - changed(copied),
71 <c>d</c> - deleted, <c>i</c> - ignored, <c>u</c> - unchanged, <c>f</c> - forced,
72 <c>w</c> - warnings, <c>c</c> - conflicts, <c>e</c> - errors, <c>y</c> - timeshifted,
73 <c>1</c> - files, <c>2</c> - directories, <c>3</c> - symlinks, <c>4</c> - uncknown,
74 <c>s</c> - short format, <c>l</c> - long format, <c>p</c> - old format,
75 <c>m..FORMAT..</c> - user format,
76 <c>z00</c> - 00 items per page,
77 <c>r..PATT..</c> - Pattern filter for paths, "wilds" or "!regexp",
78 <c>x</c> - no report.
79 <y>a #ACTION FILTER</y> - manual assign ACTION for paths meet FILTER.
80 - ACTION is [<c>bag</c>, <c>origin</c>, <c>delete</c>, <c>skip</c>, <c>restore</c>,
81 <c>tsbag</c>, <c>tsorigin</c>, <c>tsreset</c>].
82 - FILTER is "wilds" or "!regexp".
83 - # is "<c>+</c>"-interactive, "<c>-</c>" - no list, or nothing - normal.
84 <y>r</y> - Repeat report.
85 <y>y</y> - continue synchronization.
86 <y>n</y> - cancel synchronization.
87
88 """
89
90 pagemenu = u"""
91 <y>ENTER</y> - next page, "<y>NUMBER</y>" - page size, "<y>q</y>" - quit report, "<y>a</y>" - list all.
92 """
93
94
95
96 dir_root = u''
97
98 dir_files = u''
99
100 dir_db = u''
101
102 dir_dbt = u''
103
104 dir_cfg = u''
105
106 dir_cfgt = u''
109 """
110 Initialize directories.
111 """
112 global dir_root, dir_files, dir_db, dir_dbt, dir_cfg, dir_cfgt
113
114
115 p = os.path.dirname(os.path.abspath(unicode(__file__) + u''))
116 d,t = os.path.split(p)
117
118 if t in [u'pybag.py',u'pybag.pyz']:
119
120 p = d
121 else:
122
123 p = d
124 dir_root = p
125
126 dir_files = os.path.join(dir_root, u'pybagfiles')
127
128 dir_db = os.path.join(dir_root, u'pybag.db')
129
130 dir_dbt = os.path.join(dir_root, u'pybag.db~')
131
132 dir_cfg = os.path.join(dir_root, u'pybag.cfg')
133
134 dir_cfgt = os.path.join(dir_root, u'pybag.cfg~')
135
145
148 """
149 Start GUI.
150 """
151 log(u'start GUI')
152 cfg.ISGUI = True
153 try:
154 import gui
155 except Exception,exc:
156 prt0(u'<r>ERROR: When import gui - %s </r>' % (str(exc),))
157 if not init_program():
158 return
159 gui.start_gui(cmd)
160
163 """
164 Start command interface.
165 """
166 dbg(u'Main: pc.l=', cmd.l)
167 dbg(u'Main: pc.t=', cmd.t)
168 dbg(u'Main: pc.m=', cmd.m)
169 dbg(u'Main: pc.v=', cmd.v)
170 stdout_wrapper.setSkipTags(not cfg.PYBAG_COLORIZE)
171 if cmd.c and (cmd.c[-1] in [u'0', u'1']):
172 cfg.PYBAG_COLORIZE = bool(int(cmd.c[-1]))
173 stdout_wrapper.setSkipTags(not cfg.PYBAG_COLORIZE)
174 if cmd.l and (cmd.l[-1] in [u'0', u'1']):
175 cfg.PYBAG_LOG = bool(int(cmd.l[-1]))
176 if cmd.t and (cmd.t[-1] in [u'0', u'1']):
177 cfg.PYBAG_DEBUG = bool(int(cmd.t[-1]))
178 if cmd.m and (cmd.m[-1] in [u'0', u'1']):
179 cfg.PYBAG_EMUL = bool(int(cmd.m[-1]))
180 if cmd.v and (cmd.v[-1] in [u'-1', u'0', u'1', u'2', u'3', u'4', u'5']):
181 cfg.VERBOSE = int(cmd.v[-1])
182 log(u'start COMMAND="%s"' % (cmd.cmd,))
183 prt3( __header__ )
184 prt3(u'\nRun on Python %s\n' % (sys.version,))
185 if cfg.PYBAG_LOG:
186 prt1( u'Logging enabled.' )
187 if cfg.PYBAG_COLORIZE:
188 prt1( u'Colorize enabled.' )
189 if cfg.PYBAG_DEBUG:
190 prt1( u'Debug message enabled.' )
191 if cfg.PYBAG_EMUL:
192 prt1( u'Emulation mode turn on.' )
193 prt1(u'Verbosity level = ', cfg.VERBOSE)
194
195 getattr(_module, u'cmd_'+cmd.cmd, cmd_unknown)(cmd)
196 log(u'end COMMAND="%s"' % (cmd.cmd,))
197
200 """
201 Start command parser.
202
203 @return: Object B{ret} if OK, else False.
204 B{ret} contains next attributes:
205
206 - B{cmd} - command name, str.
207 - B{fs} - files list, list.
208 - B{option name} - (option, values), list.
209 @rtype: Class object or bool.
210
211 """
212 log(u'parse_command: start')
213
214 ops = u'hf:d:e:c:l:t:c:s:k:m:v:r:ao:izp:g:b:y:x:'
215 lops = [u'gui',u'add', u'dist', u'sync', u'stat', u'debug', u'help', u'usage',
216 u'remove', u'relocate', u'version', u'author', u'license', u'copyright',
217 u'about', u'cleandb']
218 args = cfg.arguments[1:]
219
220 class ret:
221 pass
222
223 ret.cmd = ''
224
225 [ setattr(ret, o, []) for o in ops if o != u':']
226
227 dbg(u'parse_command: Arguments is: ', args)
228 try:
229 opr, fsr = getopt.getopt(args, ops, lops)
230 op = []
231 fs = []
232 for o,v in opr:
233 try:
234 v = z_u(v)
235 op.append( (o, v) )
236 dbg(u'\t op "%s"="%s"' % (o,v))
237 except Exception, exc:
238 prt0(u'Error then decode options.', place=u'parse_command', error=exc)
239 for o in fsr:
240 try:
241 o = z_u(o)
242 fs.append(o)
243 dbg(u'\t fs "', o, u'"')
244 except Exception, exc:
245 prt0(u'Error then decode file list.', place=u'parse_command', error=exc)
246
247 dbg(u'parse_command: decoded options is: ', op)
248 dbg(u'parse_command: decoded files list is: ', fs)
249 except getopt.GetoptError, e:
250 prt5(u'Option Exception: ', e)
251 prt0( u'ERROR:' )
252 prt0( e.msg )
253 prt0( u'For help use: pybag.py --help' )
254 return False
255
256 ret.fs = fs
257 lops = [ u'--' + o for o in lops ]
258 dbg(u'parse_command: lops=%s' % (lops,))
259 ops = [ u'-' + o for o in ops if o != u':']
260 dbg(u'parse_command: ops=%s' % (ops,))
261 for o, v in op:
262 dbg(u'parse_command: Try parse option "%s" = "%s"' % (o,v))
263 if o in lops:
264
265 if ret.cmd:
266 prt5(u'parse_command: \tMore then one command')
267 prt0( u'Argument error: Command may be specified only once. For help use: pybag.py --help' )
268 return False
269 else:
270 dbg(u'parse_command: \tCommand recognized')
271 ret.cmd = o[2:]
272 elif o in ops:
273
274 dbg(u'parse_command: \toption recognized')
275
276 getattr(ret, o[1:]).append(v)
277 else:
278
279 prt5(u'parse_command: \toption error: Find not handled option %s' % (o,))
280 prt0( u'ERROR: Runtime error: Find not handled option %s' % (o,) )
281 return False
282
283
284 prt5(u'parse_command: *** options parse result ***')
285 prt5(u'parse_command: \tcmd="%s"' % (ret.cmd,))
286 [(prt5(u'parse_command: \topt="%s", val="%s"' % (o,getattr(ret,o[1:]))) ) for o in ops ]
287 prt5(u'parse_command: \tfile list="%s"' % (ret.fs,) )
288 log(u'parse_command: end')
289 return ret
290
293 """
294 Test if programm location already initialized.
295
296 @return: True if initialized.
297 @rtype: bool
298 """
299 r = os.path.lexists(dir_cfg)
300 r = r and os.path.isfile(dir_cfg) and not os.path.islink(dir_cfg)
301 r = r and os.path.isfile(dir_db) and not os.path.islink(dir_db)
302 r = r and os.path.isdir(dir_files) and not os.path.islink(dir_files)
303 return r
304
306 """
307 Initialize program environment.
308
309 Test current program location for initialization.
310 Ask user for initialization if need and initialize if user agree.
311 In initialization time (if user agree) set emulation off.
312 After return it in previous state.
313 Create needed files and folders if it not exist.
314 Read configuration file.
315
316 @return: True if initialization successful, otherwise False.
317 @rtype: bool
318 """
319 log(u'start INIT PROGRAM')
320
321 r = test_init()
322 e = cfg.PYBAG_EMUL
323 if not r:
324 msg = u'\nCurrent program location is not initialized.' \
325 u' For work all functions you must initialize program location.' \
326 u' (In initialization will be created needed files and folders).' \
327 u'\n\nInitialize current location ?'
328 if cfg.ISGUI:
329 from gui import ask_yn
330 r = ask_yn(msg,u'Program initialization')
331 else:
332 prt(msg, u' (y, n)\n', verbose=-100)
333 r = raw_input()
334 r = r == 'y'
335
336 if not r:
337 prt(u'<r>Program not initialized. Exit.</r>\n', verbose=-100)
338 return False
339
340 cfg.PYBAG_EMUL = False
341
342 cfg.Z_DB = {}
343 r = initpybagfiles(dir_files)
344 r = r and initconfig(dir_cfg)
345 r = r and initdb(dir_db, cfg.Z_DB)
346 cfg.PYBAG_EMUL = e
347 cfg.TSBAG = 0.0
348 cfg.TSORIG = 0.0
349 cfg.TSDIR = TSDIRNONE
350 if not r:
351 if cfg.ISGUI:
352 from gui import ask_ok
353 ask_ok(u'Program not initialized.\nFor detailes see log file.\nExit.\n',u'Program initialization')
354 else:
355 prt(u'<r>Program not initialized.\nFor details see log file.\n\nExit.</r>', verbose=-100)
356 return False
357 cfg.Z_CP.load(dir_cfg)
358
359 t = cfg.Z_CP.getopt(u'root', P_VERB, default=VERBOSE_DEFAULT)
360 if t and (t in [u'-1', u'0', u'1', u'2', u'3', u'4', u'5']):
361 cfg.VERBOSE = int(t)
362 else:
363 prt2(u'<r>Wrong configuration option "%s" value "%s".</r>' % (P_VERB,t))
364 t = cfg.Z_CP.getopt(u'root', P_COLORIZE, default=PYBAG_COLORIZE_DEFAULT)
365 if t and (t in [u'0', u'1']):
366 cfg.PYBAG_COLORIZE = bool(int(t))
367 stdout_wrapper.setSkipTags(not cfg.PYBAG_COLORIZE)
368 else:
369 prt2(u'<r>Wrong configuration option "%s" value "%s".</r>' % (P_COLORIZE,t))
370 t = cfg.Z_CP.getopt(u'root', P_DEBUG, default=PYBAG_DEBUG_DEFAULT)
371 if t and (t in [u'0', u'1']):
372 cfg.PYBAG_DEBUG = bool(int(t))
373 else:
374 prt2(u'<r>Wrong configuration option "%s" value "%s".</r>' % (P_DEBUG,t))
375 t = cfg.Z_CP.getopt(u'root', P_LOGGING, default=PYBAG_LOG_DEFAULT)
376 if t and (t in [u'0', u'1']):
377 cfg.PYBAG_LOG = bool(int(t))
378 else:
379 prt2(u'<r>Wrong configuration option "%s" value "%s".</r>' % (P_LOGGING,t))
380 t = cfg.Z_CP.getopt(u'root', P_EMUL, default=PYBAG_EMUL_DEFAULT)
381 if t and (t in [u'0', u'1']):
382 cfg.PYBAG_EMUL = bool(int(t))
383 else:
384 prt2(u'<r>Wrong configuration option "%s" value "%s".</r>' % (P_EMUL,t))
385 t = cfg.Z_CP.getopt(u'root', P_OUTFILTER, default=REPORTFILTER_DEFAULT)
386 if t and not transreport(t):
387 cfg.REPORTFILTER_DEFAULT = t
388 else:
389 prt2(u'<r>Wrong configuration option "%s" value "%s".</r>' % (P_OUTFILTER,t))
390
391 cfg.MDDELTA = float(cfg.Z_CP.getopt(u'root', P_MDDELTA, False, MDDELTA_DEFAULT))
392
393 log(u'end INIT PROGRAM')
394 return True
395
397 """
398 Initialize bag in path. Create path if need.
399 Do not change already exists files.
400
401 @param path: Path to initialize bag.
402 @type path: str
403 @return: True if successfull or False otherwise.
404 @rtype: bool
405 """
406 log(u'initpath: start')
407 prt3(u'Initialization path "%s".' % (path,), place=u'initpath')
408 try:
409 if not os.path.lexists(path):
410
411 os.makedirs(path)
412 pbf = os.path.join(path, u'pybagfiles')
413 cfgp = os.path.join(path, u'pybag.cfg')
414 dbp = os.path.join(path, u'pybag.db')
415 r = initpybagfiles(pbf)
416 r = r and initconfig(cfgp)
417 r = r and initdb(dbp)
418 except Exception, exc:
419 prt1(u'<r>Error occur then initialize path "%s".</r>' % (path,),
420 place=u'initpath',error=exc)
421 r = False
422 if not r:
423 prt3(u'Initialization failed.', place=u'initpath')
424 else:
425 prt3(u'Initialization success.', place=u'initpath')
426 log(u'initpath: end')
427 return r
428
430 """
431 Initialize database file in specified path.
432 If not exists then create DB == L{db} (by default is empty DB).
433 @param path: Path to initialize DB.
434 @type path: str
435 @param db: DB for initializing.
436 @type db: dict = {}
437 @return: True if successfull, False overwise.
438 @rtype: bool
439 """
440 log(u'initdb: start')
441
442 r = True
443 if os.path.lexists(path):
444 if os.path.isfile(path):
445 if not os.path.islink(path):
446 prt4('Database file founded.',place=u'initdb')
447 else:
448 prt0(u'<r>Database file can not be symbolic link</r>',
449 place=u'initdb')
450 r = False
451 else:
452 prt0(u'<r>Database file must be a file.</r>',
453 place=u'initdb')
454 r = False
455 elif not cfg.PYBAG_EMUL:
456
457 dbsave(db, path)
458 if os.path.lexists(path):
459 prt4(u'Database file created.', place=u'initdb')
460 else:
461 prt0(u'<r>Can not create database file</r>', place=u'initdb')
462 r = False
463 else:
464 prt0(u'Database file not exists and can\'t be created in emulation mode',
465 place=u'initdb')
466 r = False
467 log(u'initdb: end')
468 return r
469
471 """
472 Initialize configuration file in specified path.
473
474 If not exists then create next configuration file::
475
476 # PYBAG configuraton file.
477
478 [root]
479
480 nextid = 1
481 ignore =
482 compare = stat
483 symlink = copy
484 colorize = 1
485
486 # Subroots sections.
487
488 @param path: Path to initialize configuration file.
489 @type path: str
490 @return: True if successfull, False overwise.
491 @rtype: bool
492 """
493 log(u'initconfig: start')
494 defcfg = '''
495 # PYBAG configuraton file.
496
497 [root]
498
499 version = %s
500 nextid = 1
501 mddelta = 5
502 ignore =
503 compare = stat
504 symlink = copy
505
506 # Subroots sections.
507
508 ''' % __version__
509
510
511 r = True
512 if os.path.lexists(path):
513 if os.path.isfile(path):
514 if not os.path.islink(path):
515 prt4(u'Configuration file founded.',place=u'initconfig')
516 else:
517 prt0(u'<r>Configuration file can not be symbolic link</r>',
518 place=u'initconfig')
519 r = False
520 else:
521 prt0(u'<r>Configuration file must be a file.</r>',
522 place=u'initconfig')
523 r = False
524 elif not cfg.PYBAG_EMUL:
525
526 try:
527 fp = None
528 fp = codecs.open(path, 'w', 'utf8','strict')
529 fp.write(defcfg)
530 prt4(u'Configuration file created.', place=u'initconfig')
531 except Exception, exc:
532 prt0(u'<r>Can not create configuration file.</r>', place=u'initconfig', error=exc)
533 r = False
534 finally:
535 if fp:
536 fp.close()
537 else:
538 prt0(u'Config file not exists and cant be created in emulation mode',
539 place=u'initconfig')
540 r = False
541 log(u'initconfig: end')
542 return r
543
545 """
546 Initialize pybagfiles directory in specified path.
547
548 If not exists then create empty directory "pybagfiles".
549 @param path: Path to initialize pybagfiles directory.
550 @type path: str
551 @return: True if successfull, False overwise.
552 @rtype: bool
553 """
554 log(u'initpybagfiles: start')
555 r = True
556
557 if os.path.lexists(path):
558 if os.path.isdir(path):
559 if not os.path.islink(path):
560 prt4(u'Pybagfiles directory founded.', place=u'initpybagfiles')
561 else:
562 prt0(u'<r>Pybagfiles directory can not be symbolic link</r>',
563 place=u'initpybagfiles')
564 r = False
565 else:
566 prt0(u'<r>Pybagfiles must be a directory.</r>',
567 place=u'initpybagfiles')
568 r = False
569 elif not cfg.PYBAG_EMUL:
570
571 try:
572 os.mkdir(path)
573 prt4(u'Pybagfiles directory created.', place=u'initpybagfiles')
574 except Exception, exc:
575 prt0(u'<r>Can not create pybagfiles directory</r>', place=u'initpybagfiles', error=exc)
576 r = False
577 else:
578 prt0(u'Pybagfiles directory not exists and cant be created in emulation mode',
579 place=u'initpybagfiles')
580 r = False
581 log(u'initpybagfiles: end')
582 return r
583
585 """
586 Deinitialize program structures and close opened files.
587 """
588 log(u'DEINIT PROGRAM')
589
590 log_clear()
591
593 """
594 Global preferences saver class.
595 Tuned for pybag using (handle roots).
596 """
597
599 """
600 Initialize base class.
601 """
602 ConfigParser.SafeConfigParser.__init__(self)
603
604
605 - def load(self, path=None):
606 """
607 Load configuration file.
608
609 Load "pybag.cfg" from given path or from L{dir_cfg}.
610 @param path: Path to configuration file. If None (default) then use
611 L{dir_cfg} path.
612 @type path: str = None
613 """
614 if not path:
615 path = dir_cfg
616 f = None
617 try:
618 f = codecs.open(path, 'r', 'utf_8', 'strict', 0)
619 self.readfp(f)
620 if self.has_section(u'root'):
621 if self.has_option(u'root', 'mddelta'):
622 try:
623 cfg.MDDELTA = float(self.get(u'root', 'mddelta'))
624 prt5(u'Loaded configuration MDDELTA=%f.' % (cfg.MDDELTA,))
625 except Exception, exc:
626 prt0(u'<r>Can not obtain configuration MDDELTA.</r>',place=u'_gp.load', error=exc)
627 except Exception, e:
628 prt0(u'<r>Can\'t read configuration.</r>', place=u'_gp.load', error=e)
629 finally:
630 if f:
631 f.close()
632 self.dump()
633
634 - def save(self, path=None, patht=None):
635 """
636 Write configuration to file.
637
638 Also set "version" option in "root" section to current programm version.
639
640 @param path: Path to save configuration file. If None (default) then use
641 L{dir_cfg} path.
642 @type path: str = None
643 @param patht: Path to save backup configuration file. If None (default) then use
644 "path"+"~".
645 @type patht: str = None
646 @return: See L{z_r} for result.
647 """
648 log(u'_gp.save: start')
649 if not path:
650 path = dir_cfg
651 if not patht:
652 patht = z_u(path) + u'~'
653 if cfg.PYBAG_EMUL:
654 prt1(u'_gp.save: Emulation ON, do not save.')
655 else:
656
657 try:
658 if os.path.exists(patht):
659 os.remove(patht)
660 except Exception, exc:
661 prt0(u'<r>Can not remove cfg~ file.</r>', place=u'_gp.save', error=exc)
662 try:
663
664 if os.path.exists(path):
665 os.rename(path, patht)
666 except Exception, exc:
667 prt0(u'<r>Can not rename cfg to cfg~ file.</r>',
668 u' Original file will be owerwritten.', place=u'_gp.save', error=exc)
669 try:
670
671 if not self.has_section(u'root'):
672 self.add_section(u'root')
673 self.set(u'root', u'version', __version__)
674
675 dbg(u'_gp.save: Try save config.')
676 fp = codecs.open(path, 'w', 'utf8','strict')
677
678 fp.write(self.dumpstr())
679 prt4(u'cfg file successfully saved.', place=u'_gp.save')
680 except Exception, exc:
681 prt0(u'<r>Can not write cfg file.</r>', place=u'_gp.save', error=exc)
682 finally:
683 if fp:
684 fp.close()
685 self.dump()
686 log(u'_gp.save: end')
687
688 - def getroots(self, only_default=False):
689 """
690 Get list of roots name registered in config [root] section.
691 Options with name "r####" in [root] section.
692
693 @param only_default: True for get only default.
694 @type only_default: bool
695
696 @return: List of registered roots.
697 @rtype: list of str
698 """
699 log('_gp.getroots: start')
700 r = []
701 try:
702 for op in self.options(u'root'):
703 m = cfg.z_re_rootsec.match(op)
704 if m:
705 rn = m.group(0)
706
707 seldef = self.getopt(rn, P_SELECTDEFAULT, False, P_SELECTDEFAULTNO) == P_SELECTDEFAULTYES
708 if (not only_default) or seldef:
709 r.append(op)
710 except Exception, exc:
711 prt5(u'_gp.getroots: ERROR %s' % (str(exc),))
712 log(u'_gp.getroots: end')
713 return r
714
716 """
717 Get list of roots bag path registered in config [root] section.
718 """
719 ret = []
720 for p in self.getroots():
721 t = self.getrootbag(p)
722 if t:
723 ret.append(t)
724 return ret
725
727 """
728 Return roots path relative to bag and os specific.
729
730 If "rs" is list then return roots path in list.
731 If "rs" is string (one root) then return path in string.
732 Roots path return in OS specific, relative to pybagfiles.
733 If no roots then return empty list or empty string.
734 @param rs: Roots names in list or single root name in string.
735 @type rs: list or str
736 @return: Roots path in OS specific.
737 @rtype: list or str
738 """
739 log(u'getrootspath: start')
740 if type(rs) == list:
741 rss = rs
742 else:
743 rss = [rs]
744 r = []
745 for rn in rss:
746 dbg(u'getrootspath: search "%s".' % (rn,))
747 if self.has_option(u'root', rn):
748 s = path2os(splitpathsdb(self.get(u'root', rn, True).split(' ',1)[-1])[-1])
749 log(u'getrootspath: founded "%s".' % (s,))
750 r.append(s)
751 else:
752 prt1(u'Root "%s" not found.' % (rn,),place=u'getrootspath')
753 if type(rs) != list:
754 if len(r)>0:
755 r = r[0]
756 else:
757 r = u''
758 log(u'getrootspath: end')
759 return r
760
762 """
763 Return roots path.
764
765 If "rs" is list then return roots flag in list.
766 If "rs" is string (one root) then return flag in string.
767 @param rs: Roots names in list or single root name in string.
768 @type rs: list or str
769 @return: Roots flag.
770 @rtype: list or str
771 """
772 log(u'getrootflag: start')
773 if type(rs) == list:
774 rss = rs
775 else:
776 rss = [rs]
777 r = []
778 for rn in rss:
779 dbg(u'getrootsflag: search "%s".' % (rn,))
780 if self.has_option(u'root', rn):
781 s = self.get(u'root', rn, True).split(' ',1)[0]
782 log(u'getrootsflag: founded "%s".' % (s,))
783 r.append(s)
784 else:
785 prt1(u'Root "%s" not found.' % (rn,),place=u'getrootsflag')
786 if type(rs) != list:
787 if r:
788 r = r[0]
789 else:
790 r = u''
791 log(u'getrootsflag: end')
792 return r
793
795 """
796 Add root in roots list in [root] section.
797 New root value set to empty string.
798
799 @return: Name of new root.
800 @rtype: str
801 """
802 log(u'_gp.addroot: start')
803 _id = int(self.getopt('root', 'nextid', True, '0'))
804 dbg(u'_gp.addroot: get _id=%d' % (_id,))
805 rn = u'r%d' % (_id,)
806 if rn in self.getroots():
807
808 prt0(u'ERROR - Unique indexing breaked (%s)' % (rn,), place=u'_gp.addroot',
809 error=u'Unique indexing breaked')
810 rn = None
811 else:
812 _id = _id + 1
813 dbg(u'_gp.addroot: *** 1')
814 self.set(u'root', u'nextid', z_u(_id))
815 dbg(u'_gp.addroot: *** 2')
816 self.set(u'root', rn, u'')
817 log(u'_gp.addroot: end (rn="%s")' % (rn,))
818 return rn
819
821 """
822 Remove root registration and its root section.
823 If no such root then do nothing.
824
825 @param rn: Root name.
826 @type rn: str
827 """
828 log(u'_gp.delroot: start(%s)' % (rn,))
829 try:
830 if cfg.z_re_rootsec.match(rn) and (rn != u'root'):
831 if self.has_option(u'root', rn):
832
833 prt5(u'\t remove registration "%s"' % (rn,), place=u'_gp.delroot')
834 if not self.remove_option(u'root', rn):
835 prt0(u'ERROR: Can not remove root option "%s".' % (rn,), place=u'z_gp_class.delroot')
836 else:
837 prt5(u'No such option "%s"' % (rn,),place=u'_gp.delroot')
838 sn = rn + u'_prefs'
839 if self.has_section(sn):
840
841 prt5(u'\t remove prefs "%s"' % (sn,), place=u'_gp.delroot')
842 if not self.remove_section(sn):
843 prt0(u'ERROR: Can not remove root section "%s".' % (rn,), place=u'z_gp_class.delroot')
844 else:
845 prt5(u'No such section "%s"' % (sn,),place=u'_gp.delroot')
846 else:
847 prt5(u'_gp.delroot: ERROR Bad root name "%s"' % (rn,))
848 except Exception, exc:
849 prt5(u'_gp.delroot: ERROR %s' % (str(exc),))
850 log(u'_gp.delroot: end')
851
853 """
854 Find root for L{bagpath}.
855
856 @param bagpath: Bag path for syncronized files.
857 @type bagpath: str
858 @return: Root name or None if root not found.
859 @rtype: str or None
860 """
861 log(u'_gp.findroot: start (%s)' % (z_u(bagpath),))
862 r = None
863 rs = self.getroots()
864 dbg('_gp.findroot: rs="%s"' % (rs,))
865 p = splitpaths(bagpath)
866 if len(p):
867 p = path2db(p[0])
868 dbg('_gp.findroot: p="%s"' % (p,))
869 for rn in rs:
870 pr = self.getopt(u'root', rn)
871 dbg('_gp.findroot: pr="%s"' % (pr,))
872 pr = pr.split(' ',1)[-1]
873
874 pr = splitpathsdb(pr)
875
876
877 if len(pr) and pr[-1] == p:
878 r = rn
879 log(u'_gp.findroot: root finded "%s"' % (r,))
880 break
881 if not r:
882 dbg(u'_gp.findroot: root NOT found')
883 log(u'_gp.findroot: end')
884 return r
885
887 """
888 Return root bag path in os specific.
889 Return empty string if no such root name.
890 """
891 rs = self.getopt(u'root',rootname)
892 if rs:
893 rs = splitpathsdb(rs.split(' ',1)[-1])[-1]
894 rs = path2os(rs)
895 return rs
896
897 - def getopt(self, rn, opt, create=False, default=u''):
898 """
899 Get option L{opt} in root name L{rn}. If option absent in
900 B{rn} section then try search in "root" section.
901 If no option found and C{L{create} == True} then
902 create new option (and section if need) and set value to
903 default.
904 If no option found and C{L{create} == False} then return
905 L{default} value.
906
907 @param rn: Root name. May be "r####" or "root".
908 @type rn: str
909 @param opt: Option name.
910 @type opt: str
911 @param create: If C{True} then create new section and option
912 if need. Default C{False}.
913 @type create: bool = {False}
914 @param default: Default value.
915 @type default: str = { Empty string }
916 @return: Value if has or created option, overwise None.
917 @rtype str or None
918 """
919 log(u'_gp.getopt: start')
920 r = default
921 log(u'_gp.getopt: ("%s", "%s", "%s") ' % (rn,opt,default))
922 if (not cfg.z_re_rootsec.match(rn)) and (rn != u'root'):
923 prt5(u'_gp.getopt: ERROR Bad root name %s' % (rn,))
924 else:
925 if rn != u'root':
926 rn = rn + u'_prefs'
927 hs = self.has_section(rn)
928 ho = False
929 dbg(u'_gp.getopt: *** 0')
930 if hs:
931 ho = self.has_option(rn, opt)
932 if (hs and ho) or create:
933
934 dbg(u'_gp.getopt: *** 1')
935 if ( (not hs) or (not ho) ) and self.has_section(u'root') and \
936 self.has_option(u'root', opt):
937 r = self.get(u'root', opt, True)
938 else:
939
940 dbg(u'_gp.getopt: *** 2')
941 if not hs:
942 dbg(u'_gp.getopt: Add section "%s"' % (rn,))
943 self.add_section(rn)
944 if not ho:
945 dbg(u'_gp.getopt: Add option "%s->%s"="%s"' % (rn,opt,default))
946 self.set(rn, opt, default)
947 dbg(u'_gp.getopt: get opt "rn("%s")->opt("%s")' % (rn,opt))
948 try:
949 dbg(u'_gp.getopt: *** 3')
950 dbg(u'secs %s' % (self.sections()))
951 dbg(u'opts %s' % (self.options(rn)))
952
953 r = self.get(rn, opt, True)
954 dbg(u'_gp.getopt: *** 4')
955 except Exception, exc:
956 dbg(u'_gp.getopt: ERROR: ' % (exc,))
957 raise exc
958 dbg(u'_gp.getopt: r is set')
959 else:
960
961 if self.has_section(u'root') and self.has_option(u'root', opt):
962 r = self.get(u'root', opt, True)
963 log(u'_gp.getopt: end')
964 return r
965
966 - def setopt(self, rn, opt, value=''):
967 """
968 Set option L{opt} in root name L{rn}.
969 If no such option then
970 create new option (root must be registered) and set value.
971
972 @param rn: Root name. May be "r####" or "root".
973 @type rn: str
974 @param opt: Option name.
975 @type opt: str
976 @param value: Value.
977 @type value: str
978 """
979 log('_gp.setopt: start ("%s", "%s", "%s")' % (rn,opt,value))
980 rnp = rn
981 if (not cfg.z_re_rootsec.match(rn)) and (rn != u'root'):
982 prt5(u'_gp.setopt: ERROR Bad root name "%s"' % (rn,))
983 elif (rn!=u'root') and (rn not in self.getroots()):
984 prt5(u'_gp.setopt: ERROR Root not registered "%s"' % (rn,))
985 else:
986 if rnp != u'root':
987 rnp = rnp + u'_prefs'
988 if not self.has_section(rnp):
989 self.add_section(rnp)
990 dbg(u'_gp.setopt: WARNING: Add root section "%s"' % (rnp,))
991 dbg(u'_gp.setopt: set ok')
992 self.set(rnp, opt, value)
993 log(u'_gp.setopt: end')
994
996 """
997 Delete option L{opt}.
998
999 @param rn: Root name. May be "r####" or "root".
1000 @type rn: str
1001 @param opt: Option name.
1002 @type opt: str
1003 """
1004 log(u'_gp.delopt: start')
1005 if (not cfg.z_re_rootsec.match(rn)) and (rn != u'root'):
1006 prt5(u'_gp.delopt: ERROR Bad root name %s' % (rn,))
1007 else:
1008 if rn != u'root':
1009 rn = rn + u'_prefs'
1010 if self.has_section(rn):
1011 if self.has_option(rn, opt):
1012 self.remove_option(rn, opt)
1013 else:
1014 dbg(u'_gp.delopt: No option "%s" in [%s].' % (opt,rn))
1015 else:
1016 dbg(u'_gp.delopt: No section %s.' % (rn,))
1017 log(u'_gp.delopt: end')
1018
1020 """
1021 Debug dump config to log.
1022 """
1023 dbg(u'_gp.dumpstr: start')
1024 r = u''
1025 ss = self.sections()
1026 for s in ss:
1027 r += u'\n[%s]\n' % (s,)
1028 os = self.options(s)
1029 for o in os:
1030 v = self.get(s,o, True)
1031 r += u'%s = %s\n' % (o,v)
1032 dbg(u'_gp.dumpstr: end')
1033 return r
1034
1036 """
1037 Debug dump config to log.
1038 """
1039 dbg(u'_gp.dump: start')
1040 for l in self.dumpstr().split(u'\n'):
1041 dbg(u'_gp.dump:\t"%s"' % (l,))
1042 dbg(u'_gp.dump: end')
1043
1045 """
1046 Class for result return from commands.
1047
1048 @cvar l: List of returned items. Default is empty list [].
1049 @type l: list
1050 @cvar e: Error messages. If no error then empty string ''.
1051 In "e" saves only last error.
1052 @type e: str
1053 """
1054
1055 l = []
1056 e = u''
1057
1058 @staticmethod
1060 """
1061 Clear all values to default.
1062 """
1063 z_r.l = []
1064 z_r.e = u''
1065
1067 """
1068 Translate report option I{-g} into global variables.
1069 @param opt: Option value.
1070 @type opt: str
1071 @return: C{u''} - empty string if all OK. Error string otherwise.
1072 @rtype: str
1073 """
1074 log(u'transreport: start = "%s"' % opt)
1075 r = u''
1076 if opt is None:
1077 opt = REPORTFILTER_DEFAULT
1078 try:
1079 repatt = ur'^((z(?P<pagesize>\d\d))|' \
1080 ur'(?P<reportshow>x)|' \
1081 ur'(?P<dirflag>[' + u''.join(O_REPORTFILTER.keys()) + ']+)|' \
1082 ur'(?P<pathtype>[' + u''.join(O_REPORTTYPE.keys()) + ']+)|' \
1083 ur'(r(?P<rbound>..)(?P<rpatt>.+?)(?P=rbound))|' \
1084 ur'(m(?P<mbound>..)(?P<mpatt>.+?)(?P=mbound))|' \
1085 ur'(?P<format>[' + u''.join(O_REPORTFORMAT.keys()) + ']))+$'
1086 dbg(u'transreport: repatt = "%s"' % repatt)
1087 rc = re.compile(repatt, re.DOTALL | re.LOCALE | re.UNICODE)
1088 rm = rc.match(opt)
1089 if rm:
1090 d = rm.groupdict()
1091 cfg.REPORTSHOW = not bool(d[u'reportshow'])
1092 cfg.REPORTPAGE = int(d[u'pagesize'] or REPORTPAGE_DEFAULT)
1093 cfg.REPORTPATT = d[u'rpatt'] or REPORTPATT_DEFAULT
1094 cfg.REPORTTYPE = u''.join([O_REPORTTYPE[i] for i in (d[u'pathtype'] or REPORTTYPE_DEFAULT)])
1095 cfg.REPORTFILTER = 0
1096 for i in (d[u'dirflag'] or REPORTDIR_DEFAULT):
1097 cfg.REPORTFILTER |= O_REPORTFILTER[i]
1098 cfg.REPORTFORMAT = d[u'format'] or d[u'mpatt'] or REPORTFORMAT_DEFAULT
1099 if cfg.REPORTFORMAT in O_REPORTFORMAT:
1100 cfg.REPORTFORMAT = O_REPORTFORMAT[cfg.REPORTFORMAT]
1101 dbg(u'\treportpage="',cfg.REPORTPAGE, u'"\n\tREPORTPATT="',cfg.REPORTPATT,
1102 u'"\n\tREPORTTYPE="',cfg.REPORTTYPE,u'"\n\tREPORTFILTER="',cfg.REPORTFILTER,
1103 u'"\n\tREPORTFORMAT="',cfg.REPORTFORMAT,u'"')
1104
1105 cire = cfg.REPORTPATT
1106 if len(cire) > 0:
1107 if cire[0] == u'!':
1108 cire = cire[1:]
1109 else:
1110
1111 cire = cire.replace(u'\\',u'\\\\').replace(u'|',u'\\|').replace(u'.',u'\\.').replace(u'*',u'.*'). \
1112 replace(u'?',u'.?').replace(u'$',u'\\$').replace(u'^',u'\\^').replace(u'{',u'\\{'). \
1113 replace(u'(',u'\\(').replace(u'[',u'\\[').replace(u'+',u'\\+').split(u',')
1114 cire = u'|'.join([u'('+i+u'$)' for i in cire if i])
1115 if not cire:
1116 cire = u'^$'
1117 try:
1118 dbg(u'transreport: cire = "%s"' % z_u(cire))
1119 ci = re.compile(cire, re.LOCALE | re.UNICODE)
1120 cfg.REPORTPATTCMP = ci
1121 except Exception, exc:
1122 prt5(u'Error when compiling pattern string "%s".' % cire, error=exc, place=u'transreport')
1123 r = u'Error when compiling pattern string.'
1124
1125 else:
1126 r = u'Option string for report format is wrong.'
1127 except Exception,exc:
1128 r += u'Error in program:' + str(exc)
1129 prt4(u'Error in program.',place=u'transreport',error=exc)
1130 log(u'transreport: end = "%s"' % r)
1131 return r
1132
1134 """
1135 Read database file pybag.db (gzipped).
1136 If errors occurs then setup L{z_r.e} to error message.
1137 In DB files placed in next format:
1138 - DB_VERSION_MARKER DB_VERSION
1139 - F SIZE MDATE CRC32 path_to_file1_in_pybagfiles link_point1
1140 - F SIZE MDATE CRC32 path_to_file2_in_pybagfiles link_point2
1141 - ... and so on.
1142 All lines must end with newline.:
1143 - "F" - flag: d-directory, f-file, s-symlink, ''-is empty entry.
1144 - "SIZE" - file size in decimal or "-" for directories.
1145 - "MDATE" - modification date in YYYY/MM/DD-HH:mm:ss.
1146 - "CRC32" - crc32 in hex if exist or "-".
1147 - "path_to_file" - bag path to item into pybagfiles id "db" format.
1148 - "link_point" - if item link then it's point in "db" format, otherwise '-'.
1149 All lines must end with newline.
1150 Read only DB information.
1151
1152 @param path: Path from reading DB. If None then use L{dir_db}.
1153 @type path: str = None
1154 @return: DB dictionary. In L{z_r} return output and errors.
1155 In returned dbdictionary key is normalized/cased path
1156 (relative to pybag), value is dict of next items (see in pybag description).
1157 @rtype: dict
1158 """
1159 log(u'dbread: start')
1160 DBPATH_ = DBPATH
1161 DBDBFLAG_ = DBDBFLAG
1162 DBDBSIZE_ = DBDBSIZE
1163 DBDBMDATE_ = DBDBMDATE
1164 DBDBCRC32_ = DBDBCRC32
1165 DBDBREAD_ = DBDBREAD
1166 DBREADYES_ = DBREADYES
1167 DBDBLINK_ = DBDBLINK
1168 doout = cfg.PYBAG_DEBUG
1169 dbg_ = dbg
1170 if not path:
1171 path = dir_db
1172 try:
1173 log(u'dbread: db path = "%s"' % path)
1174 fp = gzip.open(path, 'rb', 9)
1175 af = fp.read()
1176 ad = af.decode(u'utf_8', u'strict')
1177 al = ad.splitlines()
1178 db = {}
1179 v = al[0]
1180 del al[0]
1181 v = v.strip()
1182 dbg(u'...line:"%s"' % (v,))
1183 lv = len(v)
1184 if v.startswith(z_db_marker) and (lv >= z_db_marker_len + 3):
1185 v = v[z_db_marker_len : z_db_marker_len + 3]
1186 else:
1187 v = None
1188 if (v == u'0.1') and (v != None):
1189 dbg_(u'dbread: Read db V 0.1')
1190 for c in al:
1191 c = c.strip()
1192 if doout:
1193 dbg_(u'.|.line:"%s"' % c)
1194 if c:
1195 try:
1196 (f, sz, md, crc, path, link) = c.split(u' ',5)
1197
1198 path = path.replace(u'#0025',u'%')
1199 db[path] = {DBPATH_:path,
1200 DBDBFLAG_:f, DBDBSIZE_:sz, DBDBMDATE_:md, DBDBCRC32_:crc,
1201 DBDBREAD_:DBREADYES_, DBDBLINK_:link}
1202 except Exception,exc:
1203 prt0(u'Error when read DB. Skip line.', place=u'dbread', error=exc)
1204 if not doout:
1205 log(u'.|.line:"%s"' % c)
1206
1207
1208
1209
1210
1211 if doout:
1212 dbg_(u'dbread: readed %s' % db[path])
1213 elif (v == u'0.2') and (v != None):
1214 dbg_(u'dbread: Read db V 0.2')
1215 for c in al:
1216 c = c.strip()
1217 if doout:
1218 dbg_(u'.|.line:"%s"' % (c,))
1219 if c:
1220 try:
1221 (f, sz, md, crc, path, link) = c.split(u' ',5)
1222
1223 db[path] = {DBPATH_:path,
1224 DBDBFLAG_:f, DBDBSIZE_:sz, DBDBMDATE_:md, DBDBCRC32_:crc,
1225 DBDBREAD_:DBREADYES_, DBDBLINK_:link}
1226 except Exception,exc:
1227 prt0(u'Error when read DB. Skip line.', place=u'dbread', error=exc)
1228 if not doout:
1229 log(u'.|.line:"%s"' % c)
1230
1231
1232
1233
1234
1235 if doout:
1236 dbg_(u'dbread: readed %s' % db[path])
1237 else:
1238
1239 prt0(u'Unsupproted DB version.', place=u'dbread', error=u'Unsupproted DB version.')
1240
1241 except Exception, exc:
1242 prt0(u'Error when read DB.', place=u'dbread', error=exc)
1243 finally:
1244 if fp:
1245 fp.close()
1246 log(u'dbread: end')
1247 return db
1248
1249 -def dbsave(dbdict, path=None, patht=None):
1250 """
1251 Save database to file.
1252 If errors occurs hen setup L{z_r.e} to error message.
1253 In DB files placed in next format:
1254 - DB_VERSION_MARKER DB_VERSION
1255 - F SIZE MDATE CRC32 path_to_file1_in_pybagfiles link_point1
1256 - F SIZE MDATE CRC32 path_to_file2_in_pybagfiles link_point2
1257 - ... and so on.
1258 For details format see L{dbread}.
1259
1260 @param path: Path from save DB. If None then use L{dir_db}.
1261 @type path: str = None
1262 @param patht: Path from save backup DB. If None then use "path"+"~".
1263 @type patht: str = None
1264 @param dbdict: DB dictionary. See L{dbread} for details.
1265 @type dbdict: dict
1266 """
1267 log(u'dbsave: start')
1268 DBDBFLAG_ = DBDBFLAG
1269 DBDBSIZE_ = DBDBSIZE
1270 DBDBMDATE_ = DBDBMDATE
1271 DBDBCRC32_ = DBDBCRC32
1272 DBDBREAD_ = DBDBREAD
1273 DBREADYES_ = DBREADYES
1274 DBDBLINK_ = DBDBLINK
1275 doout = cfg.PYBAG_DEBUG
1276 dbg_ = dbg
1277 if not path:
1278 path = dir_db
1279 if not patht:
1280 patht = z_u(path) + u'~'
1281 if cfg.PYBAG_EMUL:
1282 prt1(u'Emulation is ON do not save.', place=u'dbsave')
1283 prt1(u'DB context logged in log file.', place=u'dbsave')
1284 prt5(u'\ndbsave: DB CONTEXT START\n\n')
1285 prt5(dbdump(dbdict))
1286 prt5(u'\ndbsave: DB CONTEXT END\n\n')
1287 else:
1288 if cfg.PYBAG_DEBUG:
1289
1290 dbg(u'\ndbsave: DB CONTEXT START\n\n')
1291 dbg(dbdump(dbdict))
1292 dbg(u'\ndbsave: DB CONTEXT END\n\n')
1293
1294 try:
1295 if os.path.exists(patht):
1296 dbg(u'dbsave: \tremove backup')
1297 os.remove(patht)
1298 except Exception, exc:
1299 prt0(u'Can not remove db~ file.', place=u'dbsave', error=exc)
1300 try:
1301
1302 if os.path.exists(path):
1303 dbg(u'dbsave: \tcreate temp')
1304 os.rename(path, patht)
1305 except Exception, exc:
1306 prt0(u'Can not rename db to db~ file.'
1307 u'Original file will be owerwritten.', place=u'dbsave', error=exc)
1308 try:
1309
1310 dbg(u'dbsave: \topen gzip')
1311 fp = None
1312 fp = gzip.open(path, 'wb', 9)
1313 dbg(u'dbsave: \twrite marker')
1314 fp.write('%s%s\n\n' % (z_db_marker, z_db_version))
1315 dbg(u'dbsave: \twrite rows start')
1316 for c,o in dbdict.iteritems():
1317 s = u'%s %s %s %s %s %s\n' % (o[DBDBFLAG_], o[DBDBSIZE_],
1318 o[DBDBMDATE_], o[DBDBCRC32_], c, o[DBDBLINK_])
1319 se = s.encode(u'utf_8', u'strict')
1320 if doout:
1321 dbg_(u'dbsave: \t\twrite row ...:%s' % s)
1322 fp.write(se)
1323 except Exception, exc:
1324 prt0(u'Can not write db file.', place=u'dbsave', error=exc)
1325 finally:
1326 if fp:
1327 fp.close()
1328 log(u'dbsave: end')
1329
1331 """
1332 Dump DB row to string.
1333 Dump only items from L{z_db_formatlist}.
1334 """
1335
1336 s = u''
1337 for i,t,d,n in z_db_formatlist:
1338 if dbrow.has_key(i):
1339 try:
1340 s += z_u(tab) + u'(' + z_u(n) + u')="' + z_u(dbrow[i]) + u'"\n'
1341 except:
1342 s += z_u(tab) + u'!!!ERROR then dump DB row\n'
1343
1344 return s
1345
1346 -def dbdump(db, tab = u'DBDUMP:'):
1347 """
1348 Dump DB to string.
1349 Dump only items from L{z_db_formatlist}.
1350 """
1351 dbg(u'dbdump: start')
1352 doout = cfg.PYBAG_DEBUG
1353 s = u''
1354 dr = DBDUMPMAX
1355 for k,v in db.iteritems():
1356 s += z_u(tab) + u'[' + z_u(k) + u']:\n'
1357 s += dbdumprow(v, u'\tDBDUMP:....>')
1358 dr -= 1
1359 if dr < 0:
1360 if doout:
1361 dbg(u'dbdump: max cout restricted ... stop dumping.')
1362 s += u'\n*** max cout restricted ... stop dumping. ***\n'
1363 break
1364 s += u'\n'
1365 dbg(u'dbdump: end')
1366 return s
1367
1369 """
1370 Normalize DB - convert values to right type and past default if absent.
1371 If no such index or wrong values or type then create and set to default.
1372
1373 @return: Output and errors in L{z_r} and changed input L{db}.
1374 """
1375 log(u'dbnormalize: start')
1376 LOCK_ = cfg.LOCK
1377 LI_PERCENT_ = LI_PERCENT
1378 LI_CANCEL_ = LI_CANCEL
1379 doout = cfg.PYBAG_DEBUG
1380 _u_ = z_u
1381 dbg_ = dbg
1382 LOOPN_ = LOOPN
1383 DBBACKUP_ = DBBACKUP
1384 DBBAGFLAG_ = DBBAGFLAG
1385 DBDBFLAG_ = DBDBFLAG
1386 DBDBLINK_ = DBDBLINK
1387 DBBAGLINK_ = DBBAGLINK
1388 DBBAGMDATE_ = DBBAGMDATE
1389 DBDBMDATE_ = DBDBMDATE
1390 DBBAGPATH_ = DBBAGPATH
1391 DBDBPATH_ = DBDBPATH
1392 DBBAGREAD_ = DBBAGREAD
1393 DBDBREAD_ = DBDBREAD
1394 DBBAGSIZE_ = DBBAGSIZE
1395 DBDBSIZE_ = DBDBSIZE
1396 LN = 0
1397 for k,v in db.iteritems():
1398
1399
1400 LN += 1
1401 if LN >= LOOPN_:
1402 LOCK_[LI_PERCENT_] += LN
1403 LN = 0
1404 if LOCK_[LI_CANCEL_]:
1405 prt3(u'CANCELED.',place=u'dbnormalize')
1406 return
1407 for i,t,d,s in z_db_formatlist:
1408 if doout:
1409 dbg_(u'dbnormalize: ::: i,t,d=%s, %s, %s, %s' % (_u_(i),_u_(t),_u_(d), _u_(s) ))
1410 try:
1411 q = t(v.get(i, d))
1412 except:
1413 q = d
1414 v[i] = q
1415 LOCK_[LI_PERCENT_] += LN
1416 log(u'dbnormalize: end')
1417
1419 """
1420 Return list of path's parts.
1421
1422 @param path: Path.
1423 @type path: str
1424 @return: List of splited parts of path. If path empty then empty list.
1425 @rtype: list
1426
1427 @warning: Normalize path before splitting. See C{os.path.normpath()}.
1428 """
1429 if path:
1430 r = os.path.normpath(path)
1431
1432
1433
1434 r = r.split(cfg.OSSEP)
1435 if not r[0]:
1436 r[0] = cfg.OSSEP
1437 if cfg.OSWIN and r[0][-1] == u':':
1438 r[0] = r[0] + cfg.OSSEP
1439 if not r[-1]:
1440 del r[-1]
1441 else:
1442
1443 r = path
1444 if cfg.PYBAG_DEBUG:
1445 dbg(u'splitpaths: "%s" = %s' % (path, str(r)))
1446 return r
1447
1449 """
1450 See L{splitpaths}. Used for DB format paths.
1451 """
1452
1453 r = path.split(DBSEP)
1454 if not r[0]:
1455 r[0] = DBSEP
1456 if cfg.PYBAG_DEBUG:
1457 dbg(u'splitpathsdb: "%s" = %s' % (path, str(r)))
1458 return r
1459
1461 """
1462 For DB format paths return parent directory for given path.
1463 If no parent return empty string u''.
1464 """
1465 r = path.rsplit(DBSEP,1)[0]
1466 if r == path:
1467 r = u''
1468 if cfg.PYBAG_DEBUG:
1469 dbg(u'getpathdirdb: "%s" = %s' % (path, str(r)))
1470 return r
1471
1473 """
1474 Test if B{dbpath} starts with B{idx}.
1475 """
1476
1477 r = dbpath.startswith(idx)
1478 if r and len(dbpath) > len(idx):
1479 if dbpath[len(idx)] != DBSEP:
1480 r = False
1481 if cfg.PYBAG_DEBUG:
1482 dbg(u'dbstartswith: end ("%s", "%s")->(%s)' % (z_u(dbpath), z_u(idx),str(r)))
1483 return r
1484
1486 """
1487 Convert path to DB format (separate by L{DBSEP}).
1488 @warning: Path started from root "/" converted wrongly -
1489 "/" character changed to os separator.
1490 For example "C{/abc/def.ghi}" converted to "C{\/abc/def.ghi}" under Windows
1491 and "C{#002F/abc/def.ghi}" under unix.
1492 """
1493
1494 p = DBSEP.join([i.replace(u'#',u'#0023').replace(u'/',u'#002F').replace(u' ',u'#0020').replace( \
1495 u'\r',u'#000D').replace(u'\n',u'#000A').replace(u'\f',u'#000C').replace(u'\t',u'#0009') \
1496 for i in splitpaths(path)])
1497 if cfg.PYBAG_DEBUG:
1498 dbg(u'path2db: end ("%s")->("%s")' % (path,p))
1499 return p
1500
1502 """
1503 Convert path to OS format from DB.
1504 """
1505
1506
1507
1508
1509
1510 p = path.split(DBSEP)
1511 if not p:
1512 p = cfg.OSSEP
1513 p = os.path.join(*[i.replace(u'#0009',u'\t').replace(u'#000C',u'\f').replace(u'#000A',u'\n').replace( \
1514 u'#000D',u'\r').replace(u'#002F',u'/').replace(u'#0020',u' ').replace(u'#0023',u'#') \
1515 for i in p])
1516 if cfg.PYBAG_DEBUG:
1517 dbg(u'path2os: end ("%s")->("%s")' % (path,p))
1518 return p
1519
1520 -def walk(path, onerror=None):
1521 """
1522 Top-down directory walker.
1523 return in iterate (r,d,f), as python standard os.walk.
1524 Note: Default python os.walk some times generate error in mixing encodings
1525 in file names ... may be it bug?
1526 """
1527
1528 if not os.path.isdir(path) or os.path.islink(path):
1529 raise ValueError(u'"path" must be directory (not file or symlink.')
1530
1531 def onerr(arg, iserr):
1532 """onerror stub (quiet)."""
1533 pass
1534
1535 errfun = onerr
1536
1537 if callable(onerror):
1538 errfun = onerror
1539
1540 ds = [path]
1541
1542 fl = []
1543 dl = []
1544 r = ''
1545
1546
1547 while ds:
1548 try:
1549 r = ds.pop()
1550 dl = list()
1551 fl = list()
1552 al = os.listdir(r)
1553 for it in al:
1554 itp = os.path.join(r,it)
1555 if os.path.isdir(itp):
1556 if os.path.islink(itp):
1557 errfun(itp, False)
1558 else:
1559 dl.append(it)
1560 else:
1561 fl.append(it)
1562 yield (r, dl, fl)
1563 while dl:
1564 ds.append(os.path.join(r,dl.pop()))
1565 except Exception, exc:
1566 errfun(exc, True)
1567
1570 """
1571 Test file for symlink emulation or real.
1572 Return True if symlink, False overwise.
1573 @param useemul: If true then use symlink emulation if no native support.
1574 @type useemul: bool
1575 """
1576 doout = cfg.PYBAG_DEBUG
1577
1578 r = False
1579 if useemul is None:
1580 useemul = cfg.SYMLINKEMUL
1581
1582 fp = None
1583 try:
1584 if os.path.islink(path):
1585
1586 r = True
1587 elif useemul and os.path.isfile(path):
1588 if doout:
1589 dbg(u'symlinktest: isfile' )
1590 if os.path.getsize(path) <= z_pblmaxlen:
1591
1592 fp = codecs.open(path, 'r', 'utf_8', 'strict', 0)
1593 s = fp.read(z_pblmaxlen)
1594 s = s.strip()
1595 if doout:
1596 dbg(u'symlinktest: readed="%s"...' % (s[0:255],) )
1597 if s.startswith(z_pbls) and s.endswith(z_pble):
1598
1599 r = True
1600 except Exception, exc:
1601 dbg(u'symlinktest: Error then test link "%s"\n\terror: %s' % (path,str(exc)))
1602 finally:
1603 if fp:
1604 fp.close()
1605 if doout:
1606 dbg(u'symlinktest: end (%s)' % (z_u(r),) )
1607 return r
1608
1610 """
1611 Read symbolic link point.
1612 Use native function if support or emulate.
1613 If filesystem not support link then also emulate it.
1614
1615 Emulate file format is: "pybaglink start:xxxxxxxxxx:pybaglink end".
1616 File encoding is "utf_8".
1617
1618 @param useemul: If true then use symlink emulation if no native support.
1619 @type useemul: bool
1620
1621 @raise IOError: if error then read link.
1622 """
1623 doout = cfg.PYBAG_DEBUG
1624
1625 if useemul is None:
1626 useemul = cfg.SYMLINKEMUL
1627
1628 def rlemul():
1629
1630 fp = None
1631 try:
1632 if os.path.getsize(path) > z_pblmaxlen:
1633
1634 raise IOError, u'File "%s" is not pybag link emulation (too long).' % (path,)
1635 fp = codecs.open(path, 'r', 'utf_8', 'strict', 0)
1636 s = fp.read(z_pblmaxlen)
1637 s = s.strip()
1638
1639 if not s.startswith(z_pbls) and not s.endswith(z_pble):
1640
1641 raise IOError, u'File "%s" is not pybag link emulation.' % (path,)
1642 lnk = s[len(z_pbls):-len(z_pble)]
1643 except Exception, exc:
1644 prt3(u'Error then read link "%s".' % (path,), place=u'readlink', error = exc)
1645 raise exc
1646 finally:
1647 if fp:
1648 fp.close()
1649
1650 return lnk
1651
1652 if os.path.lexists(path):
1653 if os.path.islink(path):
1654
1655 if hasattr(os, 'readlink'):
1656
1657 pe = path.encode(cfg.z_epse, u'strict')
1658
1659 lnk = os.readlink(pe)
1660 lnk = lnk.decode(cfg.z_epse, u'strict')
1661 if doout:
1662 dbg(u'readlink: readed "%s"' % (lnk,))
1663 else:
1664
1665 raise IOError, u'OS do not support link.'
1666 elif useemul:
1667
1668 lnk = rlemul()
1669 else:
1670
1671 raise IOError, u'File "%s" is not symlink.' % (path,)
1672 else:
1673 raise IOError, u'Path not exists.'
1674
1675 if doout:
1676 dbg(u'readlink: end "%s"' % lnk)
1677 return lnk
1678
1679 -def symlink(link, path, useemul=None):
1680 """
1681 Save symbolic link point.
1682 Use native function if support or emulate.
1683 If filesystem not support link then also emulate it.
1684 Before create new link you must remove previous.
1685
1686 Emulate file format is: "pybaglink start:xxxxxxxxxx:pybaglink end".
1687 File encoding is "utf_8".
1688
1689 @param useemul: If true then use symlink emulation if no native support.
1690 @type useemul: bool
1691
1692 @raise IOError: if error then save link.
1693 """
1694
1695
1696
1697 if useemul is None:
1698 useemul = cfg.SYMLINKEMUL
1699
1700 def slemul():
1701
1702
1703
1704 fp = None
1705 try:
1706 s = u'%s%s%s' % (z_pbls,link,z_pble)
1707 if len(s) > z_pblmaxlen:
1708 prt1(u'Error then save link (too long) "%s".' % (path,), place=u'symlink', error = exc)
1709
1710 fp = codecs.open(path, 'w', 'utf_8', 'strict')
1711 fp.write(s)
1712 except Exception, exc:
1713 prt1(u'Error then save link "%s".' % (path,), place=u'symlink', error = exc)
1714 raise exc
1715 finally:
1716 if fp:
1717 fp.close()
1718
1719
1720 if not os.path.lexists(path):
1721 if hasattr(os, 'readlink'):
1722
1723 try:
1724 pe = path.encode(cfg.z_epse, u'strict')
1725
1726
1727 os.symlink(link,pe)
1728 except OSError,exc:
1729
1730 prt5(u'symlink: Can\'t use OS symlink(%s).' % (z_u(exc),), place=u'symlink', error=exc )
1731 if useemul:
1732 slemul()
1733 else:
1734 raise exc
1735 else:
1736
1737
1738 if useemul:
1739 slemul()
1740 else:
1741 raise IOError, u'Symlink not supported by OS.'
1742 else:
1743 raise IOError, u'Path already exists.'
1744
1747 """
1748 Delete directory recursively.
1749 Delete everything reachable from the directory named in 'path',
1750 (do not follow for symbolic link).
1751 @warning: This is dangerous! For example, if path == '/home', it
1752 could delete all your home files.
1753
1754 B{!!! ROOT FOLDERS ('/' and 'C:') CAN NOT BE DELETED THIS FUNCTION!!!}
1755
1756 B{If you want delete root folder you must delete it manualy.}
1757 """
1758 log(u'rmdirr: start')
1759 dbg_ = dbg
1760 ospath_ = os.path
1761 opj_ = ospath_.join
1762 osrm_ = os.remove
1763 osrmd_ = os.rmdir
1764 os_ = os
1765 if os.path.normpath(path.upper()) in [u'/', u'C:', u'C:/', u'C:\\']:
1766 prt0(u'ROOT FOLDER "%s" CAN NOT BE DELETED!!! Please delete it manualy.' % path)
1767 return
1768 def onerror(err):
1769 """
1770 Logout error when walking.
1771 """
1772 prt2('FAIL WALK read pathfile: "%s"' % (z_u(err),), place='rmdirr')
1773
1774
1775 for root, dirs, files in os_.walk(path, False, onerror):
1776 dbg_( u'rmdirr: \twalk root "%s".' % (root,) )
1777 for name in files:
1778 pn = opj_(root, name)
1779 dbg_(u'rmdirr: delete file "%s".' % pn)
1780 if not cfg.PYBAG_EMUL:
1781 try:
1782 osrm_(pn)
1783 except Exception, exc:
1784 prt5(u'Error delete file "%s".' % (pn,), place=u'rmdirr',error=exc)
1785 for name in dirs:
1786 pn = opj_(root, name)
1787 dbg_(u'rmdirr: delete dir "%s".' % pn)
1788 if not cfg.PYBAG_EMUL:
1789 try:
1790 osrmd_(pn)
1791 except Exception, exc:
1792 prt5(u'Error delete folder "%s".' % (pn,), place=u'rmdirr',error=exc)
1793 if not cfg.PYBAG_EMUL:
1794 try:
1795 if os.path.isdir(path):
1796 osrmd_(path)
1797 else:
1798 osrm_(path)
1799 except Exception, exc:
1800 prt5(u'Error delete folder "%s".' % (path,), place=u'rmdirr',error=exc)
1801 log(u'rmdirr: end')
1802 return
1803
1804 -def copytree(src, dst, symlinks=False,exclude=[]):
1805 """
1806 Copy files tree from "src" to "dst". "dst" must be a directory.
1807 If "symlinks" True then also copy
1808 symlinks (default False).
1809 If symlinks not supported then they emulated.
1810 If "dst" not exists then create it.
1811 From copying exluded items equal "exclude" list.
1812 """
1813 log(u'copytree: start')
1814 if cfg.PYBAG_EMUL:
1815 prt3(u'Emulation is on - do not copytree.')
1816 return True
1817 try:
1818
1819 if not os.path.lexists(dst):
1820 os.makedirs(dst)
1821 elif not os.path.isdir(dst):
1822 prt1(u'Path "%s" must be a directory.' % (dst,), place=u'copytree')
1823 return False
1824 if os.path.isdir(src):
1825
1826 ld = os.listdir(src)
1827 for name in ld:
1828 prt5(u'\t try copy "%s".' % (name,),place=u'copytree')
1829 if name in exclude:
1830 prt5(u'\texclude "%s".' % (name,),place=u'copytree')
1831 continue
1832 try:
1833 srcname = os.path.join(src, name)
1834 dstname = os.path.join(dst, name)
1835 if symlinks and symlinktest(srcname):
1836 linkto = readlink(srcname)
1837 symlink(linkto, dstname)
1838 elif os.path.isdir(srcname):
1839 copytree(srcname, dstname, symlinks,exclude)
1840 else:
1841 shutil.copy2(srcname, dstname)
1842 except Exception, exc:
1843 prt1(u'Can\'t copy "%s" to "%s".' % (srcname, dstname),place=u'copytree',error=exc)
1844 return False
1845 else:
1846
1847 if src in exclude:
1848 prt5(u'\texclude "%s".' % (src,),place=u'copytree')
1849 return True
1850 if symlinks and symlinktest(src):
1851 linkto = readlink(src)
1852 symlink(linkto, dst)
1853 else:
1854 shutil.copy2(src, dst)
1855 except Exception, exc:
1856 prt1(u'Can\'t copy "%s" to "%s".' % (src, dst),place=u'copytree',error=exc)
1857 return False
1858 log(u'copytree: end success')
1859 return True
1860
1862 """
1863 Validate paths.
1864
1865 @param root: Root (r1, r2, ...).
1866 @type root: str
1867 @param bagpath: Root path in pybag. If None then get from root config.
1868 @type bagpath: str { = None }
1869 @param syncpath: Syncronization path. If empty then used base (config) path.
1870 @type syncpath: str { = None }
1871 @return: list [root, bagpath, syncpath, dbindex] or None if path wrong.
1872 All path returned in absolute form, I{dbindex} is normalized.
1873 @rtype: list or None
1874 """
1875 log(u'validatepaths: start')
1876 r = None
1877
1878 if not bagpath:
1879 bagpath = path2os(splitpathsdb(cfg.Z_CP.getopt(u'root', root).split(' ',1)[-1])[-1])
1880 dbindex = path2db(bagpath)
1881 dbg(u'validatepaths: dbindex="%s"' % (dbindex,))
1882 bp = os.path.abspath(os.path.normpath(os.path.join(dir_files, bagpath)))
1883 dbg(u'validatepaths: bp="%s"' % (bp,))
1884 if not syncpath:
1885 dbg(u'validatepaths: path2os="%s"' % (path2os(cfg.Z_CP.getopt(u'root', root).split(' ',1)[-1])))
1886 syncpath = os.path.abspath(path2os(cfg.Z_CP.getopt(u'root', root).split(' ',1)[-1]))
1887 (syncpath, _) = os.path.split(syncpath)
1888 dbg(u'validatepaths: split syncpath head = "%s"' % (syncpath,))
1889 syncpath = os.path.join(syncpath, bagpath)
1890 dbg(u'validatepaths: join syncpath with bagpath = "%s"' % (syncpath,))
1891
1892 syncpath = os.path.abspath(os.path.normpath(syncpath))
1893 dbg(u'validatepaths: syncpath="%s"' % (syncpath,))
1894 bpe = os.path.lexists(bp)
1895 spe = os.path.lexists(syncpath)
1896 if spe or bpe:
1897 dbg(u'validatepaths: spe or bpe.')
1898 r = [root, bp, syncpath, dbindex]
1899 if not spe:
1900 prt3(u'WARNING: Origin path %s not exists.' % (syncpath,), place=u'validatepaths')
1901 if not bpe:
1902 prt3(u'WARNING: Bag path %s not exists.' % (bp,), place=u'validatepaths')
1903 else:
1904
1905 prt0(u'Paths "%s", "%s" not exists!!!' %(bp, syncpath), place='validatepaths',
1906 error=u'Paths "%s", "%s" not exists!!!' %(bp, syncpath))
1907 log(u'validatepaths: end')
1908 return r
1909
1911 """
1912 Calculate CRC32 for given file.
1913 Return 0 for other situation (directory, link, absent).
1914 """
1915 log(u'getcrc32: start ("%s")' % (path,))
1916 c = 0
1917
1918 if (os.path.lexists(path) and os.path.isfile(path) and
1919 not symlinktest(path) ):
1920
1921 fp = None
1922 try:
1923 fp = open(path, u'rb')
1924 p = True
1925 while p:
1926 p = fp.read(CRC32BUF)
1927 c = binascii.crc32(p, c)
1928 except Exception,exc:
1929 prt0(u'Can not calculate CRC32, error is occur.', place=u'getcrc32', error=exc)
1930 c = 0
1931 finally:
1932 if fp:
1933 fp.close()
1934 log(u'getcrc32: end ("0x%08X")' % (c,))
1935 return c
1936
1938 """
1939 Walk throw bagpath files from pybagfiles and syncpath for original files.
1940
1941 @param db: Database dictionary.
1942 @type db: dict
1943 @param root: Root name for read configuration.
1944 @type root: str
1945 @param path: Absolut path for walking. Path must exists.
1946 @type path: str
1947 @param dbindex: Index into DB equivalent path.
1948 @type dbindex: str
1949 @param isbag: True if walk throw bag, False for origin.
1950 @type isbag: bool
1951 @return: (success, ignored) tuple and changed input L{db}:
1952 - C{success} - bool, True if Ok, False if error or path not exists.
1953 - C{ignored} - int, number of ignored items.
1954 @rtype: tuple
1955 @warning: path must be in absolute normalized format and must exists.
1956 Usually this function get arguments that returned by L{validatepaths}.
1957 @raise ValueError: Raise if path is not absolut.
1958 """
1959 log(u'dbwalkfiles: start')
1960
1961 doout = cfg.PYBAG_DEBUG
1962 doout4 = (cfg.VERBOSE >=4 ) or cfg.PYBAG_DEBUG
1963 dbg_ = dbg
1964 prt4_ = prt4
1965 LI_PERCENT_ = LI_PERCENT
1966 LOCK_ = cfg.LOCK
1967 LI_CANCEL_ = LI_CANCEL
1968 IBAG_ = IBAG
1969 path2db_ = path2db
1970 DBSYNCERRREADBAG_ = DBSYNCERRREADBAG
1971 DBSYNCERRREADSYNC_ = DBSYNCERRREADSYNC
1972 ISYNC_ = ISYNC
1973 OSSEP_ = cfg.OSSEP
1974 OSASEP_ = cfg.OSASEP
1975 DBSEP_ = DBSEP
1976 IPATH_ = IPATH
1977 DBCOMPARE_ = DBCOMPARE
1978 DBSYMLINK_ = DBSYMLINK
1979 DBSYMLINKEMUL_ = DBSYMLINKEMUL
1980 DBBACKUP_ = DBBACKUP
1981 IREAD_ = IREAD
1982 DBREADYES_ = DBREADYES
1983 DBFLAGSYMLINK_ = DBFLAGSYMLINK
1984 DBFLAGDIR_ = DBFLAGDIR
1985 DBFLAGFILE_ = DBFLAGFILE
1986 DBFLAGUNKNOWN_ = DBFLAGUNKNOWN
1987 IFLAG_ = IFLAG
1988 DBNONE_ = DBNONE
1989 ospathgetsize = os.path.getsize
1990 ISIZE_ = ISIZE
1991 ospathgetmtime = os.path.getmtime
1992 DBMDATENONE_ = DBMDATENONE
1993 IMDATE_ = IMDATE
1994 DBLINKNONE_ = DBLINKNONE
1995 readlink_ = readlink
1996 ILINK_ = ILINK
1997 ICRC32_ = ICRC32
1998 DBSYNC_ = DBSYNC
1999 ospathsplit = os.path.split
2000 P_SYMLINK_IGNORE_ = P_SYMLINK_IGNORE
2001 DBDIRREADED_ = DBDIRREADED
2002 DBDIRIGNORED_ = DBDIRIGNORED
2003
2004 prt3(u' ... "%s".' % path, place=u'dbwalkfiles')
2005 rtn = True
2006 log(u'dbwalkfiles: root="%s", path="%s", isbag="%s", dbi="%s"'%
2007 (z_u(root), z_u(path), z_u(isbag), z_u(dbindex)))
2008 if not os.path.isabs(path):
2009
2010 raise ValueError, 'Path for walking must be absolut.'
2011
2012 ign = 0
2013
2014
2015
2016
2017 def onerror(err, iserr=True):
2018 """
2019 Logout error when walking.
2020 """
2021 if iserr:
2022 dbg_(u'walkfiles onerror called')
2023 try:
2024 prt2(u'FAIL WALK read pathfile: "%s"' % (z_u(err.filename),), place='dbwalkfiles')
2025 prt2(u'FAIL WALK read pathfile, error is ', unicode(err, errors=u'replace'))
2026 except Exception, exc:
2027 prt2(u'FAIL WALK read pathfile: "????", error in onerror.', place='dbwalkfiles')
2028 else:
2029 prt3(u'WALK ignore link to directory: "%s"' % (z_u(err),), place='dbwalkfiles')
2030
2031 def readpath(path):
2032 """
2033 Read path information into DB.
2034 Never raise exception.
2035 @return: Number ignored items (0 or 1).
2036 """
2037 if doout:
2038 dbg_(u'dbwalkfiles.readpath: start(%s)' % (path,))
2039 ignore = 0
2040 LOCK_[LI_PERCENT_] += 1
2041 if LOCK_[LI_CANCEL_]:
2042 log(u'dbwalkfiles.readpath: canceled.')
2043 return ignore
2044 try:
2045
2046 ip1 = path[plen:]
2047 if ip1.startswith(OSSEP_) or ip1.startswith(OSASEP_ or '***noaltsep***'):
2048 ip1 = ip1[1:]
2049 ip = path2db_(ip1)
2050
2051 if isbag:
2052 IG = IBAG_
2053 IRE = DBSYNCERRREADBAG_
2054 else:
2055 IG = ISYNC_
2056 IRE = DBSYNCERRREADSYNC_
2057
2058 if ip.startswith(DBSEP_):
2059 ip = ip[1:]
2060 if ip:
2061 idb = DBSEP_.join([dbindex, ip])
2062 else:
2063 idb = dbindex
2064
2065 if not db.has_key(idb):
2066 dbidb = {}
2067 else:
2068 dbidb = db[idb]
2069
2070 dbidb[IG + IPATH_] = path
2071
2072 dbidb[DBCOMPARE_] = cc
2073 dbidb[DBSYMLINK_] = cs
2074 dbidb[DBSYMLINKEMUL_] = cse
2075
2076 dbidb[IG + IREAD_] = DBREADYES_
2077 try:
2078
2079 if symlinktest(path):
2080 df = DBFLAGSYMLINK_
2081 elif os.path.isdir(path):
2082 df = DBFLAGDIR_
2083 elif os.path.isfile(path):
2084 df = DBFLAGFILE_
2085 else:
2086
2087 prt2('FAIL file "%s" is not dir, file symlink. Ignored.' % (path,), place='dbwalkfiles.readpath')
2088
2089 df = DBFLAGUNKNOWN_
2090
2091
2092
2093 dbidb[IG + IFLAG_] = df
2094
2095 dsz = DBNONE_
2096 if df == DBFLAGFILE_:
2097 dsz = ospathgetsize(path)
2098 dbidb[IG + ISIZE_] = dsz
2099
2100
2101 dmd = DBMDATENONE_
2102 if df == DBFLAGFILE_:
2103 dmd = ospathgetmtime(path)
2104 dbidb[IG + IMDATE_] = dmd
2105
2106
2107 smp = DBLINKNONE_
2108 if df == DBFLAGSYMLINK_:
2109 smp = readlink_(path)
2110 dbidb[IG + ILINK_] = smp
2111
2112
2113 dcrc32 = DBNONE_
2114
2115 dbidb[IG + ICRC32_] = dcrc32
2116
2117
2118 if doout:
2119 dbg_(u'dbwalkfiles.readpath: ip="%s"\n ip2db="%s"\n idb="%s"\n df="%s"\n' \
2120 u' dsz="%s"\n dmd="%s"\n smp="%s"\n crc32="%s"' % (z_u(ip1), z_u(ip), \
2121 z_u(idb), z_u(df), z_u(dsz), z_u(dmd), z_u(smp), z_u(dcrc32)))
2122
2123 except Exception, exc:
2124 prt2(u'FAIL in file "%s" - %s.' % (path,z_u(exc)), place='dbwalkfiles.readpath', error=exc)
2125 dbidb[DBSYNC_] |= IRE
2126
2127
2128 (_, ipi) = ospathsplit(path)
2129 if ci.match(path) or \
2130 (ipi and ci.match(ipi)) or \
2131 (df == DBFLAGUNKNOWN_) or ((cs in P_SYMLINK_IGNORE_) and (df == DBFLAGSYMLINK_)):
2132 if doout4:
2133 prt4_( u'\tIGNORED "%s".' % (path),place=u'dbwalkfiles.readpath:')
2134 ignore = 1
2135
2136 dbidb[DBDIRREADED_] = DBDIRIGNORED_
2137
2138 except Exception, exc:
2139 prt2(u'FAIL in file "%s" - %s.' % (path,z_u(exc)), place='dbwalkfiles.readpath', error=exc)
2140 db[idb] = dbidb
2141 return ignore
2142
2143
2144
2145
2146 try:
2147
2148 cire = cfg.Z_CP.getopt(root, P_IGNORE, False, P_DEFIGNORE)
2149 if len(cire) > 0:
2150 if cire[0] == u'!':
2151 cire = cire[1:]
2152 else:
2153
2154 cire = cire.replace(u'\\',u'\\\\').replace(u'|',u'\\|').replace(u'.',u'\\.').replace(u'*',u'.*'). \
2155 replace(u'?',u'.?').replace(u'$',u'\\$').replace(u'^',u'\\^').replace(u'{',u'\\{'). \
2156 replace(u'(',u'\\(').replace(u'[',u'\\[').replace(u'+',u'\\+').split(u',')
2157 cire = u'|'.join([u'('+i+u'$)' for i in cire if i])
2158 if not cire:
2159 cire = u'^$'
2160 try:
2161 ci = re.compile(cire, re.LOCALE | re.UNICODE)
2162 except Exception, exc:
2163 prt0(u'Error when compiling regexp "%s".' % cire, error=exc)
2164 raise
2165 cs = cfg.Z_CP.getopt(root, P_SYMLINK, False, P_DEFSYMLINK)
2166 cc = cfg.Z_CP.getopt(root, P_COMPARE, False, P_DEFCOMPARE)
2167 cse = cs in P_SYMLINK_EMUL
2168 cfg.SYMLINKEMUL = cse
2169 plen = len(path)
2170 dbg(u'dbwalkfiles: cire,cs,cc,cse,plen="%s", "%s", "%s", "%s", "%s"' %
2171 (z_u(cire),z_u(cs),z_u(cc),z_u(cse),z_u(plen)))
2172
2173 if os.path.lexists(path):
2174 if os.path.isfile(path) or os.path.islink(path):
2175
2176
2177
2178 if LOCK_[LI_CANCEL]:
2179
2180 log(u'dbwalkfiles: canceled')
2181 rtn = False
2182 return (rtn, ign)
2183 ign += readpath(path)
2184 else:
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199 fse = cfg.z_epse
2200 try:
2201 pathstr = path.encode(fse)
2202 except Exception, exc:
2203 prt2(u'Can\'t encode path "%s" to filesystem encoding(%s)' % (path,fse),
2204 place='dbwalkfiles', error=exc)
2205 raise
2206 for (r,d,f) in os.walk(pathstr, True, onerror):
2207 try:
2208 runi = r.decode(fse)
2209 except Exception, exc:
2210 prt2(u'Can\'t decode r "%s" from filesystem encoding(%s), break' %
2211 (unicode(path,fse,'replace'),fse), place='dbwalkfiles', error=exc)
2212
2213 break
2214
2215
2216
2217 if LOCK_[LI_CANCEL]:
2218
2219 log(u'dbwalkfiles: canceled')
2220 rtn = False
2221 return (rtn, ign)
2222 ign += readpath(runi)
2223
2224
2225
2226
2227
2228 for p in d[:]:
2229 try:
2230 puni = p.decode(fse)
2231 except Exception, exc:
2232 prt2(u'Can\'t decode d "%s" from filesystem encoding(%s), skip' %
2233 (unicode(p,fse,'replace'),fse), place='dbwalkfiles', error=exc)
2234
2235 continue
2236
2237
2238
2239 if LOCK_[LI_CANCEL]:
2240
2241 log(u'dbwalkfiles: canceled')
2242 rtn = False
2243 return (rtn, ign)
2244 if ci.match(puni):
2245 d.remove(p)
2246
2247
2248
2249 ign += readpath(os.path.join(runi, puni))
2250
2251
2252
2253 for p in f:
2254 try:
2255 puni = p.decode(fse)
2256 except Exception, exc:
2257 prt2(u'Can\'t decode f "%s" from filesystem encoding(%s), skip' %
2258 (unicode(p,fse,'replace'),fse), place='dbwalkfiles', error=exc)
2259
2260 continue
2261
2262
2263
2264 if LOCK_[LI_CANCEL]:
2265
2266 log(u'dbwalkfiles: canceled')
2267 rtn = False
2268 return (rtn, ign)
2269 ign += readpath(os.path.join(runi, puni))
2270 else:
2271 prt3(u'WARNING: Path "%s" not exists.' % (path,))
2272 rtn = True
2273
2274 except Exception, exc:
2275 prt0(u'Global error then walk throw files.', place='dbwalkfiles', error=exc)
2276 rtn = False
2277
2278 prt4(u'\t ignored = %d items.' % ign, place=u'dbwalkfiles')
2279 if not rtn:
2280 prt2(u'WARNING: Not all items readed from path "%s"' % path)
2281 return (rtn, ign)
2282 log(u'dbwalkfiles: end')
2283
2285 """
2286 Translate option in command.
2287 Get from "cmd" list "cmd.x" where x - is "opt".
2288 Get value from dictionary O_X_DICT[O_X_LIST[cmd.x[-1]]].
2289 If no "cmd.x" then return O_X_DICT[O_X_LIST[0]].
2290 If not supported option then return empty string ''.
2291 @param opt: Option symbol: 'k', 's'.
2292 @type opt: str
2293 @param cmd: Command object.
2294 @type cmd: object
2295 @return: Translated option value or empty string.
2296 @rtype: str
2297 """
2298 log(u'start transopt(%s)' % (opt,))
2299 r = ''
2300 o = getattr(cmd, opt, [])
2301 ls = getattr(_module, 'O_' + opt.upper() + '_LIST', None)
2302 di = getattr(_module, 'O_' + opt.upper() + '_DICT', None)
2303 if ls and di:
2304 if o and o[-1] in ls:
2305 ov = di[o[-1]]
2306 else:
2307 ov = di[ls[0]]
2308 prt4(u'For option %s used default = "%s"' % (opt, z_u(ov)), place=u'transopt')
2309 r = ov
2310 else:
2311 dbg(u'transopt: WRONG option "%s"' % (opt,))
2312 log(u'transopt: Option %s value %s' % (opt, z_u(r)))
2313 log(u'end f_transopt')
2314 return r
2315
2316
2317
2318 -def cmd_(cmd):
2319 """
2320 No command, only option.
2321 """
2322 log(u'start cmd_')
2323 if len(cmd.h)>0:
2324 cmd_usage(cmd)
2325 else:
2326 cmd_unknown(cmd)
2327 z_r.clear()
2328 log(u'end cmd_')
2329
2331 """
2332 Unknown command. Print message about unknown command.
2333 """
2334 prt0(u'<r>Unknown command "%s".\nuse --help, -h or --usage for help.</r>' % (cmd.cmd,))
2335 z_r.clear()
2336
2338 """
2339 Show info about bag files.
2340
2341 @return: String with statistics.
2342 """
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352 log(u'f_stat: start')
2353 DBDBSIZE_ = DBDBSIZE
2354 DBFLAGSYMLINK_ = DBFLAGSYMLINK
2355 DBDBFLAG_ = DBDBFLAG
2356 DBFLAGDIR_ = DBFLAGDIR
2357 DBFLAGFILE_ = DBFLAGFILE
2358 LOCK_ = cfg.LOCK
2359 LI_PERCENT_ = LI_PERCENT
2360 LI_CANCEL_ = LI_CANCEL
2361 dbstartswith_ = dbstartswith
2362 doout = cfg.PYBAG_DEBUG
2363
2364 info = u'\n'
2365 infocancel = u'\nOperation canceled.\n'
2366 fcs = 0
2367 dcs = 0
2368 scs = 0
2369 ics = 0
2370 iss = 0
2371 uis = 0
2372
2373 try:
2374 LOCK_[LI_DESC] = u'Calculate statistics.'
2375 LOCK_[LI_PERCENT100] = 100
2376 LOCK_[LI_PERCENT] = 0
2377
2378 cfg.Z_DB = dbread()
2379 LOCK_[LI_PERCENT100] = len(cfg.Z_DB) * 2
2380 dbnormalize(cfg.Z_DB)
2381 if LOCK_[LI_CANCEL]:
2382 log(u'f_stat: canceled.')
2383 return infocancel
2384
2385 rs = []
2386 if cmd.r:
2387
2388 prt1(u'\n<W>Get specified roots only.</W>\n')
2389 for r in cmd.r:
2390 tmp = cfg.Z_CP.findroot(r)
2391 if tmp:
2392 rs.append(tmp)
2393 else:
2394 prt1(u'<r>Root "%s" not found.</r>' % (r,),place=u'f_dist')
2395 else:
2396 if cmd.x:
2397 x = cmd.x[0]
2398 if x == u'default':
2399
2400 prt1(u'\n<W>Get default roots only.</W>\n')
2401 rs = cfg.Z_CP.getroots(True)
2402 else:
2403 if x != u'all':
2404 prt1(u'<r>Option "x" is wrong="%s". Use all roots.</r>' % (x,),place=u'f_dist')
2405
2406 prt1(u'\n<W>Get all roots.</W>\n')
2407 rs = cfg.Z_CP.getroots()
2408 else:
2409
2410 prt1(u'\n<W>Get all roots.</W>\n')
2411 rs = cfg.Z_CP.getroots()
2412
2413
2414 dbg(u'f_stat: roots is %s.' % (str(rs),))
2415 rd = {}
2416 for rn in rs:
2417
2418 rd[rn] = {}
2419 rd[rn]['rp'] = splitpathsdb(cfg.Z_CP.get(u'root', rn).split(' ',1)[-1])[-1]
2420 dbg(u'f_stat: rn = "%s", rp = "%s".' % (rn,rd[rn]['rp']))
2421 rd[rn]['fc'] = 0
2422 rd[rn]['dc'] = 0
2423 rd[rn]['sc'] = 0
2424 rd[rn]['ic'] = 0
2425 rd[rn]['is'] = 0
2426 rd[rn]['ui'] = 0
2427
2428 for dbi in cfg.Z_DB.keys():
2429
2430 LOCK_[LI_PERCENT_] += 1
2431 if LOCK_[LI_CANCEL_]:
2432 log(u'f_stat: canceled.')
2433 return infocancel
2434 vdb = cfg.Z_DB[dbi]
2435 for rn in rs:
2436 rp = rd[rn]['rp']
2437 if doout:
2438 dbg(u'f_stat: startswith ("%s","%s").' % (dbi, rp))
2439 if dbstartswith_(dbi, rp):
2440
2441
2442 rdrn = rd[rn]
2443 rdrn['ic'] = rdrn['ic'] + 1
2444 ics += 1
2445 f = vdb[DBDBFLAG_]
2446 if f == DBFLAGDIR_:
2447 rdrn['dc'] = rdrn['dc'] + 1
2448 dcs += 1
2449 elif f == DBFLAGFILE_:
2450 rdrn['fc'] = rdrn['fc'] + 1
2451 fcs += 1
2452 elif f == DBFLAGSYMLINK_:
2453 rdrn['sc'] = rdrn['sc'] + 1
2454 scs += 1
2455 else:
2456 rdrn['ui'] = rdrn['ui'] + 1
2457 uis += 1
2458 rdrn['is'] = rdrn['is'] + vdb[DBDBSIZE_]
2459 iss += vdb[DBDBSIZE_]
2460
2461
2462
2463
2464
2465 for rn in rs:
2466 rns = u'<g>Root: "%s":</g>' % path2os(rd[rn]['rp'])
2467 info += rns + u'\n' + u'-'*len(rns) + u'\n\n'
2468 info += u'\tFiles count = %d.\n' % rd[rn]['fc']
2469 info += u'\tDirectories count = %d.\n' % rd[rn]['dc']
2470 info += u'\tSymlinks count = %d.\n' % rd[rn]['sc']
2471 info += u'\tItems count = %d.\n' % rd[rn]['ic']
2472 info += u'\tItems size = %d.\n' % rd[rn]['is']
2473 if rd[rn]['ui'] > 0:
2474 info += u'\t <r>ERROR: Uncknown items count = %d.\n</r>' % (rd[rn]['ui'],)
2475 info += u'\n'
2476
2477 rns = u'<y>Summary for roots above:</y>'
2478 info += rns + u'\n' + u'='*len(rns) + u'\n\n'
2479 info += u'\tRoots count = %d.\n' % len(rs)
2480 info += u'\tFiles count = %d.\n' % fcs
2481 info += u'\tDirectories count = %d.\n' % dcs
2482 info += u'\tSymlinks count = %d.\n' % scs
2483 info += u'\tItems count = %d.\n' % ics
2484 info += u'\tItems size = %d.\n' % iss
2485 if uis > 0:
2486 info += u'\t ERROR: Unknown items count = %d.\n' % (uis,)
2487
2488 except Exception, exc:
2489 prt0(u'<r>ERROR: Programm error.</r>',place = u'f_stat', error=exc)
2490 info = u'ERROR in programm:\n\n' + traceback.format_exc()
2491 log(u'f_stat: end')
2492 return info
2493
2495 """
2496 Distribute PYBAG program to specified location.
2497 """
2498 log(u'f_dist: start')
2499 z_r.clear()
2500
2501
2502 if cmd.i and not cmd.fs:
2503 prt3(u'Initialize current location "%s".' % (dir_root,), place=u'f_dist')
2504 if not cfg.PYBAG_EMUL:
2505 initpath(dir_root)
2506
2507 cfg.Z_CP.load()
2508 rs = []
2509 if cmd.a:
2510
2511 rs = cfg.Z_CP.getroots()
2512 elif cmd.r:
2513
2514 for r in cmd.r:
2515 tmp = cfg.Z_CP.findroot(r)
2516 if tmp:
2517 rs.append(tmp)
2518 else:
2519 prt1(u'<r>Root "%s" not found.</r>' % (r,),place=u'f_dist')
2520 elif cmd.x:
2521
2522 if cmd.x[0] == u'default':
2523
2524 rbs = cfg.Z_CP.getroots()
2525 prt1()
2526 for rn in rbs:
2527 rbn = cfg.Z_CP.getrootbag(rn)
2528 seldef = cfg.Z_CP.getopt(rn, P_SELECTDEFAULT, False, P_SELECTDEFAULTNO)
2529 seldef = seldef==P_SELECTDEFAULTYES
2530 if seldef:
2531 rs.append(rn)
2532 prt1(u'\tRoot "%s" marked for synchronization.' % rbn)
2533
2534 elif cmd.x[0] == u'all':
2535 rs = cfg.Z_CP.getroots()
2536 else:
2537 prt1(u'\n<r>Option x is wrong. Do not dist roots.</r>',place=u'f_dist')
2538 log(u'roots=%s' %(rs,))
2539 if cfg.PYBAG_EMUL:
2540 prt3(u'Emulation is on - do not distribute.')
2541 return
2542
2543 if not cmd.fs:
2544 return
2545 for p in cmd.fs:
2546 try:
2547 prt3(u'Work with path "%s".' % (p,))
2548 if not os.path.lexists(p):
2549 os.makedirs(p)
2550 elif not os.path.isdir(p):
2551 prt1(u'Path "%s" must be a directory.' % (p,), place=u'f_dist')
2552 continue
2553 shutil.copy2(os.path.join(dir_root,u'pybag.py'), os.path.join(p,u'pybag.py'))
2554 except Exception, exc:
2555 prt1(u'Can\'t copy "pybag.py" from "%s" to "%s".' % (dir_root, p),place=u'f_dist',error=exc)
2556 continue
2557 if cmd.i:
2558 initpath(p)
2559 if cmd.z:
2560
2561 r = copytree(dir_root, p,False,[u'pybagfiles',u'.svn',u'pybag.py',u'pybag.cfg',
2562 u'pybag.db', u'pybag.db~', u'pybag.cfg~'])
2563
2564 cfgp = os.path.join(p,u'pybag.cfg')
2565 if not ( os.path.lexists(cfgp) and os.path.isfile(cfgp) ):
2566 prt3(u'In path "%s" configuration file not found.' % (p,))
2567 continue
2568 nc = z_gp_class()
2569 nc.load(cfgp)
2570 for rn in rs:
2571 rp = cfg.Z_CP.getrootspath(rn)
2572 rf = cfg.Z_CP.getrootsflag(rn)
2573 dbg(u'f_dist: \t root="%s", rp="%s".' % (rn,rp))
2574 if not rp or not rf:
2575 prt0(u'ERROR in program - this rn="%s" must not to be.' % (rn,),place=u'f_dist')
2576 continue
2577 try:
2578 if nc.findroot(rp):
2579 prt1(u'In path "%s" root "%s" already exists. Scip.' % (p,rp))
2580 continue
2581 nrn = nc.addroot()
2582 if not nrn:
2583 prt1(u'Can not add root to path "%s".' % (p,))
2584 continue
2585
2586 ni = cfg.Z_CP.getopt(rn, P_IGNORE, False, P_DEFIGNORE)
2587 ns = cfg.Z_CP.getopt(rn, P_SYMLINK, False, P_DEFSYMLINK)
2588 ncmp = cfg.Z_CP.getopt(rn, P_COMPARE, False, P_DEFCOMPARE)
2589 nbkp = cfg.Z_CP.getopt(rn, P_BACKUP, False, P_DEFBACKUP)
2590
2591 nc.setopt(u'root', nrn, rf + u' ' + path2db(os.path.join(dir_files, rp)))
2592 nc.setopt(nrn, P_IGNORE, ni)
2593 nc.setopt(nrn, P_SYMLINK, ns)
2594 nc.setopt(nrn, P_COMPARE, ncmp)
2595 nc.setopt(nrn, P_BACKUP, nbkp)
2596 prt3(u'Root "%s" successfull added to path "%s".' % (rp,p))
2597 except Exception, exc:
2598 prt1(u'Can\'t configure for path "%s" and root "%s".' % (p, rp),place=u'f_dist',error=exc)
2599 continue
2600 nc.save(cfgp)
2601
2602
2603 log(u'f_dist: end')
2604
2606 """
2607 Distribute PYBAG program to specified location.
2608 """
2609 log(u'start cmd_dist')
2610 f_dist(cmd)
2611 if z_r.e:
2612 prt0(u'<r>ERROR occur.</r>', place=u'cmd_dist', error=z_r.e)
2613 z_r.clear()
2614 log(u'end cmd_dist')
2615
2617 """
2618 Show statistic about bag files.
2619 """
2620 log(u'start cmd_stat')
2621 tm = time.time()
2622
2623 info = f_stat(cmd)
2624 if z_r.e:
2625 prt0(u'<r>ERROR occur.</r>', place=u'cmd_dist', error=z_r.e)
2626 z_r.clear()
2627 prt0(info)
2628 tm = time.time() - tm
2629 prt0(u'\nStatistic duration is %.3f sec.' % tm)
2630 log(u'end cmd_stat')
2631
2636
2638 """
2639 Print usage information.
2640 """
2641 log(u'start cmd_usage')
2642 f_usage(cmd)
2643 z_r.clear()
2644 log(u'end cmd_usage')
2645
2647 """
2648 Show version.
2649 """
2650 log(u'cmd_version: start')
2651 prt0(__version__)
2652 log(u'cmd_version: end')
2653
2655 """
2656 Show author.
2657 """
2658 log(u'cmd_author: start')
2659 prt0(__author__)
2660 log(u'cmd_author: end')
2661
2663 """
2664 Show license.
2665 """
2666 log(u'cmd_license: start')
2667 prt0(__license__)
2668 log(u'cmd_license: end')
2669
2671 """
2672 Show copyright.
2673 """
2674 log(u'cmd_copyright: start')
2675 prt0(__copyright__)
2676 log(u'cmd_copyright: end')
2677
2679 """
2680 Show about.
2681 """
2682 log(u'cmd_about: start')
2683 prt0(__about__)
2684 log(u'cmd_about: end')
2685
2687 """
2688 """
2689 f_usage(cmd)
2690
2692 """
2693 Print usage information.
2694 """
2695 log(u'start cmd_help')
2696 cmd_usage(cmd)
2697 z_r.clear()
2698 log(u'end cmd_help')
2699
2701 """
2702 """
2703 log(u'start cmd_add')
2704 f_add(cmd)
2705 if z_r.e:
2706 prt0( u'<r>ERROR occur:</r>',error= z_r.e)
2707 z_r.clear()
2708 log(u'end cmd_add')
2709
2711 """
2712 """
2713 log(u'start f_add')
2714 z_r.clear()
2715 pm = []
2716 if cmd.r:
2717 for p in cmd.r:
2718 try:
2719 l = os.listdir(p)
2720 l = [os.path.join(p, i) for i in l]
2721 dbg(u'f_add: recurse="%s">> "%s".' % (p,z_u(l)))
2722 pm.extend(l)
2723 except Exception,exc:
2724 prt1(u'Error then try recurse "%s".' % (p,), place=u'f_add', error=exc)
2725 if cmd.fs:
2726 pm.extend(cmd.fs)
2727 if not pm:
2728 prt0(u'<y>No paths for adding given.</y>', place=u'f_add')
2729
2730 k = transopt(u'k', cmd)
2731 s = transopt(u's', cmd)
2732 e = ''
2733 back = (cmd.b and (cmd.b[0] in [P_BACKUPNO, P_BACKUPYES]) and cmd.b[0]) or P_BACKUPNO
2734 seldef = (cmd.x and (cmd.x[0] in [P_SELECTDEFAULTNO, P_SELECTDEFAULTYES]) and cmd.x[0]) or P_SELECTDEFAULTYES
2735 if cmd.e:
2736 e = ','.join(cmd.e)
2737
2738 for p in pm:
2739 p = z_u(p)
2740 try:
2741 prt5(u'\t<g>Adding p="%s".</g>' %(p,), place=u'cmd_add')
2742 if not os.path.lexists(p):
2743 prt0(u'<y>Path "%s" not exist.</y>' % (p,), place=u'f_add')
2744 continue
2745 if p in (cfg.OSSEP,cfg.OSASEP):
2746 prt0(u'<r>Can\'t add root "%s".</r>' % (p,), place=u'f_add')
2747 continue
2748
2749 _, t = os.path.split(p)
2750
2751 if not t:
2752 prt1(u'<y>Path "%s" have no tail.</y>' % (p,), place=u'f_add')
2753 continue
2754 if cfg.Z_CP.findroot(t):
2755 prt1(u'<r>Path "%s" (root name "%s") already in bag.</r>'
2756 % (p, t), place=u'f_add')
2757 continue
2758 dbg(u'f_add: Test for path type')
2759 if symlinktest(p):
2760 flag = DBFLAGSYMLINK
2761 elif os.path.isdir(p):
2762 flag = DBFLAGDIR
2763 elif os.path.isfile(p):
2764 flag = DBFLAGFILE
2765 else:
2766 prt0(u'<r>Path "%s" has wrong type (not dir, file or symlink).</r>', place=u'f_add')
2767 continue
2768 rn = cfg.Z_CP.addroot()
2769 if not rn:
2770 break
2771
2772 if len(e)>1 and e[0]==u'!':
2773 try:
2774 re.compile(e[1:], re.LOCALE | re.UNICODE)
2775 except Exception, exc:
2776 prt0(u'<r>Error in regular expression.</r>', error=exc)
2777 raise
2778 pdb = path2db(p)
2779 dbg(u'f_add: pdb = "%s"' % (pdb,))
2780 cfg.Z_CP.setopt(u'root', rn, u'%s %s' % (flag, pdb))
2781 cfg.Z_CP.setopt(rn, P_SYMLINK, k)
2782 cfg.Z_CP.setopt(rn, P_COMPARE, s)
2783 cfg.Z_CP.setopt(rn, P_IGNORE, e)
2784 cfg.Z_CP.setopt(rn, P_BACKUP, back)
2785 cfg.Z_CP.setopt(rn, P_SELECTDEFAULT, seldef)
2786 prt3(u'<g>Added "%s".</g>' % (p,), place=u'f_add')
2787 except Exception, exc:
2788 z_r.e = z_u(exc)
2789 prt0(u'<r>Error then path "%s" add.</r>' % (z_u(p),), place=u'f_add', _error=exc)
2790 cfg.Z_CP.save()
2791 log(u'end f_add')
2792
2794 """
2795 Remove roots from bag.
2796 """
2797 log(u'cmd_remove start')
2798 if not cmd.fs:
2799 prt0(u'<y>No paths for removing given.</y>', place=u'cmd_remove')
2800 cfg.Z_DB = dbread()
2801 for p in cmd.fs:
2802 prt3(u'\tTry remove root "%s"' % (p,), place=u'cmd_remove')
2803 p = z_u(p)
2804 rn = cfg.Z_CP.findroot(p)
2805 if not rn:
2806 prt1(u'<y>Root "%s" not found in bag.</y>'
2807 % (p,), place=u'cmd_remove')
2808 continue
2809
2810 rv = validatepaths(rn, None, None)
2811 if rv:
2812 dbg(u'cmd_remove: rv')
2813 dbi = rv[3]
2814 bp = rv[1]
2815 dbg(u'cmd_remove: Test for path type')
2816 try:
2817 if os.path.exists(bp):
2818 if os.path.isdir(bp):
2819
2820 prt4(u'\t\tRemove directory.')
2821 if not cfg.PYBAG_EMUL:
2822 shutil.rmtree(bp)
2823 else:
2824
2825 prt4(u'\t\tRemove file or symlink.')
2826 if not cfg.PYBAG_EMUL:
2827 os.remove(bp)
2828 else:
2829 log(u'\t\t<y>path "%s" not exists, skip.</y>' % bp)
2830 except Exception, exc:
2831 prt0(u'\t\t<r>ERROR occur then remomving "%s".\n\t\t DB will not be deleted.</r>' % bp,
2832 place=u'cmd_remove', error=exc)
2833 continue
2834 else:
2835 dbg(u'cmd_remove: not rv')
2836
2837 bagpath = path2os(splitpathsdb(cfg.Z_CP.getopt(u'root', rn).split(' ',1)[-1])[-1])
2838 dbi = path2db(bagpath)
2839 dbg(u'cmd_remove: not rv dbi = "%s".' % dbi)
2840 prt3(u'\t\t<c>Clear DB ...</c>')
2841 ks = cfg.Z_DB.keys()
2842 for k in ks:
2843 if dbstartswith(k, dbi):
2844 prt4(u'\t\t\tclear db (%s)' % (k,), place=u'cmd_remove')
2845 del cfg.Z_DB[k]
2846 prt3(u'\t\t<c>Clear configuration file ...</c>')
2847
2848 dbg(u'cmd_remove: Z_CP.delroot')
2849 cfg.Z_CP.delroot(rn)
2850 prt3(u'\t\t<c>Root "%s" is removed.</c>' % (p,))
2851 prt3(u'<g>Removing is complete.</g>')
2852 dbsave(cfg.Z_DB)
2853 cfg.Z_CP.save()
2854 z_r.clear()
2855 log(u'cmd_remove end')
2856
2858 """
2859 Remove not exists paths/roots from DB.
2860 """
2861
2862 LOCK_ = cfg.LOCK
2863 LI_PERCENT_ = LI_PERCENT
2864 LI_CANCEL_ = LI_CANCEL
2865
2866 infocancel = u'\nOperation canceled.\n'
2867
2868 log(u'cmd_cleandb start')
2869
2870 cfg.Z_DB = dbread()
2871 ai = 0
2872 ci = 0
2873
2874 rbp = cfg.Z_CP.getrootsbag();
2875 for i in rbp:
2876 log(u'....rbp='+i)
2877
2878 ks = cfg.Z_DB.keys()
2879
2880 LOCK_[LI_DESC] = u'Calculate statistics.'
2881 LOCK_[LI_PERCENT100] = 100
2882 LOCK_[LI_PERCENT] = 0
2883
2884 cfg.Z_DB = dbread()
2885 LOCK_[LI_PERCENT100] = len(cfg.Z_DB) * 2
2886
2887 if LOCK_[LI_CANCEL]:
2888 log(u'cmd_cleandb: canceled.')
2889 return infocancel
2890
2891 for k in ks:
2892 log(u'k==> "%s"' % (k,))
2893 ai = ai + 1
2894 f = False
2895 for i in rbp:
2896 if dbstartswith(k, i):
2897 f = True
2898 if not f:
2899 del cfg.Z_DB[k]
2900 prt3(u'\tdb del "%s".' % (k,))
2901 ci = ci + 1
2902 LOCK_[LI_PERCENT_] += 1
2903 if LOCK_[LI_CANCEL_]:
2904 log(u'cmd_cleandb: canceled.')
2905 return infocancel
2906
2907
2908
2909 dbsave(cfg.Z_DB)
2910 z_r.clear()
2911 prt2(u'\tall items = %d' % (ai,))
2912 prt2(u'\tcleared items = %d' % (ci,))
2913 log(u'cmd_cleandb end')
2914
2916 """
2917 Relocate specified root (change its origin into configuration file).
2918 """
2919 log(u'cmd_relocate start')
2920 if not cmd.r:
2921 prt1(u'<r>You must specify option "-r".</r>', place=u'cmd_relocate')
2922 else:
2923 for r in cmd.r:
2924
2925 try:
2926 k = tuple(r.split(u'=',1))
2927 if len(k) != 2:
2928 prt1(u'<r>Invalid argument "%s".</r>' % (k,), place=u'cmd_relocate')
2929 continue
2930 k,v = k
2931 fr = cfg.Z_CP.findroot(k)
2932 if not fr:
2933 prt1(u'<r>Root "%s" not exists.</r>' % (k,), place=u'cmd_relocate')
2934 continue
2935 pr = cfg.Z_CP.getopt(u'root', fr)
2936 dbg('cmd_relocate: pr="%s"' % (pr,))
2937 flag = pr.split(' ',1)[0]
2938 dbg('cmd_relocate: flag="%s"' % (flag,))
2939
2940
2941 dbg(u'cmd_relocate: Check path')
2942 if not os.path.lexists(v):
2943 prt1(u'<r>Path "%s" for root "%s" not exists.</r>' % (v,k), place=u'cmd_relocate')
2944 continue
2945 if symlinktest(v):
2946 f = DBFLAGSYMLINK
2947 elif os.path.isdir(v):
2948 f = DBFLAGDIR
2949 elif os.path.isfile(v):
2950 f = DBFLAGFILE
2951 else:
2952 prt1(u'<r>Path "%s" has wrong type (not dir, file or symlink).</r>', place=u'cmd_relocate')
2953 continue
2954 if f != flag:
2955 prt1(u'<r>Path "%s" has wrong type-"%s" not equal root-"%s".</r>' % (v,f,flag), place=u'cmd_relocate')
2956 continue
2957
2958
2959 pdb = path2db(v)
2960 dbg(u'cmd_relocate: pdb = "%s"' % (pdb,))
2961 cfg.Z_CP.setopt(u'root', fr, u'%s %s' % (flag, pdb))
2962 prt3(u'<g>Root "%s" relocated to "%s".</g>' % (k,v))
2963 except Exception, exc:
2964 prt0(u'<r>Error then relocate "%s".</r>' % (r,), place=u'cmd_relocate',error=exc)
2965
2966 if not cfg.PYBAG_EMUL:
2967 cfg.Z_CP.save()
2968 else:
2969 prt0(u'Emulation is on. Do not save.',place=u'cmd_relocate')
2970 log(u'cmd_relocate end')
2971
2973 """
2974 Output comparison report from database B{db} in console.
2975
2976 Filter output
2977 """
2978 log(u'outreport: start.')
2979 tm = time.time()
2980 if not db:
2981 db = cfg.Z_DB
2982 if not ks:
2983 ks = cfg.Z_KS
2984
2985 DBDBFLAG_ = DBDBFLAG
2986 DBDIR_ = DBDIR
2987 DBDBPATH_ = DBDBPATH
2988 DBDIRERROR_ = DBDIRERROR
2989 DBDIRWARNING_ = DBDIRWARNING
2990 DBDIRCONFLICT_ = DBDIRCONFLICT
2991 DBDIRFORCED_ = DBDIRFORCED
2992 DBDBMDATE_ = DBDBMDATE
2993 DBSYMLINK_ = DBSYMLINK
2994 DBDBCRC32_ = DBDBCRC32
2995 DBDESC_ = DBDESC
2996 DBDESCMAN_ = DBDESCMAN
2997 DBDIRBAG_ = DBDIRBAG
2998 DBDIRORIG_ = DBDIRORIG
2999 DBDIRDB_ = DBDIRDB
3000 DBDIRCOPY_ = DBDIRCOPY
3001 DBDIRDELETE_ = DBDIRDELETE
3002 DBDIRNEW_ = DBDIRNEW
3003 DBDIRIGNORED_ = DBDIRIGNORED
3004 DBDIRUNCHANGED_ = DBDIRUNCHANGED
3005 DBDBMDATE_ = DBDBMDATE
3006 DBDBCRC32_ = DBDBCRC32
3007 DBDBSIZE_ = DBDBSIZE
3008 DBDBLINK_ = DBDBLINK
3009 DBBAGMDATE_ = DBBAGMDATE
3010 DBBAGFLAG_ = DBBAGFLAG
3011 DBBAGCRC32_ = DBBAGCRC32
3012 DBBAGSIZE_ = DBBAGSIZE
3013 DBBAGLINK_ = DBBAGLINK
3014 DBORIGMDATE_ = DBORIGMDATE
3015 DBORIGFLAG_ = DBORIGFLAG
3016 DBORIGCRC32_ = DBORIGCRC32
3017 DBORIGSIZE_ = DBORIGSIZE
3018 DBORIGLINK_ = DBORIGLINK
3019 DBSYMLINK_ = DBSYMLINK
3020 tsdirbag_ = cfg.TSDIR == TSDIRBAG
3021 tsdirorig_ = cfg.TSDIR == TSDIRORIG
3022 DBDIRTS_ = DBDIRTS
3023 dbdirbagorig = DBDIRBAG_ | DBDIRORIG_
3024 dbdirnewbag = DBDIRNEW_ | DBDIRBAG
3025 dbdirneworig = DBDIRNEW_ | DBDIRORIG_
3026 dbdircopybag = DBDIRCOPY_ | DBDIRBAG_
3027 dbdircopyorig = DBDIRCOPY_ | DBDIRORIG_
3028 dbdirerrconf = DBDIRERROR_ | DBDIRCONFLICT_
3029 _u_ = z_u
3030 prt0_ = prt0
3031
3032
3033 REPORTPAGE_ = cfg.REPORTPAGE
3034
3035 REPORTPATTCMP_ = cfg.REPORTPATTCMP
3036 REPORTTYPE_ = cfg.REPORTTYPE
3037 REPORTFILTER_ = cfg.REPORTFILTER
3038 REPORTFORMAT_ = cfg.REPORTFORMAT
3039 log(u'outreport: REPORTFILTER="%s"' % REPORTFILTER_)
3040 log(u'outreport: REPORTTYPE="%s"' % REPORTTYPE_)
3041 log(u'outreport: REPORTPATT="%s"' % cfg.REPORTPATT)
3042 page = 0
3043 d = {}
3044 tfmt = cfg.TIMEFORMAT
3045 for k in ks:
3046 vdb = db[k]
3047 try:
3048
3049
3050 if (REPORTFILTER_ & vdb[DBDIR_]) and (vdb[DBDBFLAG_] in REPORTTYPE_) and \
3051 not (REPORTPATTCMP_.match(vdb[DBDBPATH_]) is None):
3052
3053
3054
3055
3056
3057
3058
3059
3060 d[u'err'] = (vdb[DBDIR_] & DBDIRERROR_ and u'<r>e</r>') or \
3061 (vdb[DBDIR_] & DBDIRWARNING_ and u'<y>w</y>') or \
3062 (vdb[DBDIR_] & DBDIRCONFLICT_ and u'<Y>c</Y>') or u' '
3063 d[u'forced'] = (vdb[DBDIR_] & DBDIRFORCED_ and u'<C>!</C>' or u' ')
3064 d[u'path'] = vdb[DBDBPATH_]
3065 d[u'desc'] = vdb[DBDESC_] + vdb[DBDESCMAN_]
3066 d[u'type'] = vdb[DBDBFLAG_] or u' '
3067 if vdb[DBDBFLAG_] == DBFLAGFILE:
3068 d[u'mdatedb'] = _u_(vdb[DBDBMDATE_])
3069 d[u'gmdatedb'] = time.strftime(tfmt, time.gmtime(vdb[DBDBMDATE_]))
3070 d[u'lmdatedb'] = time.strftime(tfmt, time.localtime(vdb[DBDBMDATE_]))
3071 else:
3072 d[u'mdatedb'] = u'-'
3073 d[u'gmdatedb'] = u'-'
3074 d[u'lmdatedb'] = u'-'
3075 if vdb[DBBAGFLAG_] == DBFLAGFILE:
3076 d[u'mdatebag'] = str(vdb[DBBAGMDATE_])
3077 d[u'gmdatebag'] = time.strftime(tfmt, time.gmtime(vdb[DBBAGMDATE_]))
3078 d[u'lmdatebag'] = time.strftime(tfmt, time.localtime(vdb[DBBAGMDATE_]))
3079 else:
3080 d[u'mdatebag'] = u'-'
3081 d[u'gmdatebag'] = u'-'
3082 d[u'lmdatebag'] = u'-'
3083 if vdb[DBORIGFLAG_] == DBFLAGFILE:
3084 d[u'mdateorigin'] = str(vdb[DBORIGMDATE_])
3085 d[u'gmdateorigin'] = time.strftime(tfmt, time.gmtime(vdb[DBORIGMDATE_]))
3086 d[u'lmdateorigin'] = time.strftime(tfmt, time.localtime(vdb[DBORIGMDATE_]))
3087 else:
3088 d[u'mdateorigin'] = u'-'
3089 d[u'gmdateorigin'] = u'-'
3090 d[u'lmdateorigin'] = u'-'
3091 d[u'crc32db'] = u'0x%08X' % vdb[DBDBCRC32_]
3092 d[u'sizedb'] = _u_(vdb[DBDBSIZE_])
3093 d[u'crc32bag'] = u'0x%08X' % vdb[DBBAGCRC32_]
3094 d[u'sizebag'] = _u_(vdb[DBBAGSIZE_])
3095 d[u'crc32origin'] = u'0x%08X' % vdb[DBORIGCRC32_]
3096 d[u'sizeorigin'] = _u_(vdb[DBORIGSIZE_])
3097 d[u'symlinkdb'] = vdb[DBDBLINK_]
3098 d[u'symlinkbag'] = vdb[DBBAGLINK_]
3099 d[u'symlinkorigin'] = vdb[DBORIGLINK_]
3100 vdbdir_ = vdb[DBDIR_]
3101 if vdbdir_ & DBDIRTS_:
3102 if tsdirbag_:
3103 f = u'b'
3104 unch = u'<gw><==</gw>'
3105 elif tsdirorig_:
3106 f = u'o'
3107 unch = u'<bw>==></bw>'
3108 else:
3109 f = u'#'
3110 unch = u'<Rw>###</Rw>'
3111 else:
3112 f = u' '
3113 unch = u'==='
3114 d[u'timeshift'] = f
3115 d[u'dir'] = (vdbdir_ & DBDIRUNCHANGED_ and unch) or \
3116 (vdbdir_ & DBDIRIGNORED_ and u'---') or \
3117 (vdbdir_ & DBDIRDELETE_ and u'<r>xxx</r>') or \
3118 (vdbdir_ & DBDIRDB_ and not (vdbdir_ & dbdirbagorig) and u'===') or \
3119 ((vdbdir_ & dbdirnewbag == dbdirnewbag) and u'<G><<<</G>') or \
3120 ((vdbdir_ & dbdirneworig == dbdirneworig) and u'<C>>>></C>') or \
3121 ((vdbdir_ & dbdircopybag == dbdircopybag) and u'<g><--</g>') or \
3122 ((vdbdir_ & dbdircopyorig == dbdircopyorig) and u'<c>--></c>') or \
3123 (vdbdir_ & dbdirerrconf and u'<y>***</y>') or \
3124 u'<R>###</R>'
3125 if REPORTPAGE_ and (page >= REPORTPAGE_):
3126 tmp = raw_input(pagemenu)
3127 tmp = z_u(tmp,cfg.Z_INEP)
3128 if tmp == u'a':
3129
3130 page = REPORTPAGE_ = 0
3131 elif tmp == u'q':
3132
3133 break
3134 elif tmp == u'':
3135
3136 page = 0
3137 else:
3138 try:
3139 REPORTPAGE_ = int(tmp)
3140 page = 0
3141 except:
3142 page = REPORTPAGE_ - 1
3143 prt0_()
3144 prt0_(REPORTFORMAT_ % d)
3145 page += 1
3146
3147 except Exception, exc:
3148 prt0(u'<R>Error in report function for "%s".</R>' % vdb[DBDBPATH_], error=exc, place=u'outreport')
3149 tm = time.time() - tm
3150 prt4(u'\noutreport time is %.3f' % tm)
3151 log(u'outreport: end')
3152
3153 -def do_action(action, _filter, interactive=True, listpaths=True):
3154 """
3155 Perform changes into db[DBDIR] to specified action.
3156
3157 @param action: Action string L{O_ACTION}.
3158 @type action: basestr
3159 @param _filter: Filter string, "wilds" or "!regexp". If not
3160 specified then used C{"!.*"} - match all paths.
3161 @type _filter: string.
3162 @param interactive: If True then ask from user confirmation.
3163 @type interactive: bool
3164 @param listpaths: If True then list all affected paths.
3165 @type listpaths: bool
3166 @return: String with error or empty string if all ok.
3167 @rtype: stirng.
3168 """
3169 log(u'do_action: start ("%s", "%s", "%s", "%s")' % (action, _filter,interactive,listpaths))
3170 tm = time.time()
3171
3172 db = cfg.Z_DB
3173 ks = cfg.Z_KS
3174 dbstartswith_ = dbstartswith
3175 DBDBPATH_ = DBDBPATH
3176 DBPATH_ = DBPATH
3177 DBDBFLAG_ = DBDBFLAG
3178 DBBAGFLAG_ = DBBAGFLAG
3179 DBORIGFLAG_ = DBORIGFLAG
3180 DBFLAGDIR_ = DBFLAGDIR
3181 DBBAGREAD_ = DBBAGREAD
3182 DBORIGREAD_ = DBORIGREAD
3183 DBDBREAD_ = DBDBREAD
3184 DBDIRMAN_ = DBDIRMAN
3185 DBDIR_ = DBDIR
3186 DBDIRNONE_ = DBDIRNONE
3187 DBDESCMAN_ = DBDESCMAN
3188 DBDESCNONE_ = DBDESCNONE
3189 dbbagdel = DBDIRBAG | DBDIRDELETE | DBDIRFORCED
3190 dbdbdel = DBDIRDB | DBDIRDELETE | DBDIRFORCED
3191 dborigdel = DBDIRORIG | DBDIRDELETE | DBDIRFORCED
3192 dbbag = DBDIRBAG | DBDIRDB | DBDIRCOPY | DBDIRFORCED
3193 dbunchanged = DBDIRUNCHANGED | DBDIRFORCED
3194
3195 dborig = DBDIRORIG | DBDIRDB | DBDIRCOPY | DBDIRFORCED
3196 listpaths = listpaths or interactive
3197 verb = (listpaths and -100) or 5
3198 doout = cfg.PYBAG_DEBUG
3199
3200 AANS = [u'']
3201
3202 def inter(v):
3203 """
3204 """
3205 rrr = a = True
3206
3207 if cfg.VERBOSE >= verb:
3208 prt(u' %s [%s]' % (action, v[DBDBPATH]), verbosity=verb)
3209 if interactive:
3210 while a:
3211 a = False
3212 if AANS[0]:
3213 ans = AANS[0]
3214 else:
3215 ans = raw_input(u'y/n/Y/N? ')
3216 ans = ans.strip()
3217 if ans == u'n':
3218 rrr = False
3219 elif ans == u'N':
3220 rrr = False
3221 AANS[0] = u'n'
3222 elif ans == u'Y':
3223 AANS[0] = u'y'
3224 elif ans == u'y':
3225 pass
3226 else:
3227 prt(u'<r>Wrong answer.</r>', verbosity=-100)
3228 a = True
3229 if not rrr:
3230 log(u' %s canceled [%s]' % (action, v[DBDBPATH]))
3231 return rrr
3232
3233
3234 def act(v):
3235 """
3236 """
3237 if doout:
3238 dbg(u'act: start.')
3239 rdbag = v[DBBAGREAD_]
3240 rdorig = v[DBORIGREAD_]
3241 rddb = v[DBDBREAD_]
3242 if inter(v):
3243 if action == u'tsbag':
3244 try:
3245 rd = v[DBBAGREAD_] and v[DBDBREAD_] and v[DBORIGREAD_] and (v[DBDBFLAG_]==DBFLAGFILE)
3246 mdb = v[DBDBMDATE] - v[DBBAGMDATE]
3247 mdo = v[DBDBMDATE] - v[DBORIGMDATE]
3248 if abs(mdb) <= cfg.MDDELTA:
3249 mdb = 0.0
3250 if abs(mdo) <= cfg.MDDELTA:
3251 mdo = 0.0
3252 if rd:
3253 if mdb:
3254 cfg.TSBAG = mdb
3255 cfg.TSORIG = 0.0
3256 cfg.TSDIR = TSDIRBAG
3257 elif mdo:
3258 cfg.TSORIG = mdo
3259 cfg.TSBAG = 0.0
3260 cfg.TSDIR = TSDIRBAG
3261 else:
3262
3263 cfg.TSBAG = 0.0
3264 cfg.TSORIG = 0.0
3265 cfg.TSDIR = TSDIRNONE
3266 actionts.append(1)
3267 prt4(u'Set TSBAG=%f, TSORIG=%f, TSDIR=%s.' % (cfg.TSBAG, cfg.TSORIG, [u'None',u'Bag',u'Origin'][cfg.TSDIR]))
3268 except Exception, exc:
3269 log(u'do_action: tsbag ERROR: %s' % str(exc))
3270 elif action == u'tsorigin':
3271 try:
3272 rd = v[DBBAGREAD_] and v[DBDBREAD_] and v[DBORIGREAD_] and (v[DBDBFLAG_]==DBFLAGFILE)
3273 mdb = v[DBDBMDATE] - v[DBBAGMDATE]
3274 mdo = v[DBDBMDATE] - v[DBORIGMDATE]
3275 if abs(mdb) <= cfg.MDDELTA:
3276 mdb = 0.0
3277 if abs(mdo) <= cfg.MDDELTA:
3278 mdo = 0.0
3279 if rd:
3280 if mdb:
3281 cfg.TSBAG = mdb
3282 cfg.TSORIG = 0.0
3283 cfg.TSDIR = TSDIRORIG
3284 elif mdo:
3285 cfg.TSORIG = mdo
3286 cfg.TSBAG = 0.0
3287 cfg.TSDIR = TSDIRORIG
3288 else:
3289
3290 cfg.TSBAG = 0.0
3291 cfg.TSORIG = 0.0
3292 cfg.TSDIR = TSDIRNONE
3293 actionts.append(1)
3294 prt4(u'Set TSBAG=%f, TSORIG=%f, TSDIR=%s.' % (cfg.TSBAG, cfg.TSORIG, [u'None',u'Bag',u'Origin'][cfg.TSDIR]))
3295 except Exception, exc:
3296 log(u'do_action: tsorigin ERROR: %s' % str(exc))
3297 elif action == u'tsreset':
3298
3299 cfg.TSBAG = 0.0
3300 cfg.TSORIG = 0.0
3301 cfg.TSDIR = TSDIRNONE
3302 actionts.append(1)
3303 prt4(u'Reset time shift.')
3304 else:
3305 if v[DBDIRMAN_] == DBDIRNONE_:
3306
3307 v[DBDIRMAN_] = v[DBDIR_]
3308 v[DBDESCMAN_] = u' Manualy forced: '
3309 if action == u'delete':
3310
3311
3312 v[DBDIR_] = DBDIRNONE_
3313 if rdbag:
3314 v[DBDIR_] = dbbagdel
3315 v[DBDESCMAN_] += u'Delete in bag.'
3316 if rdorig:
3317 v[DBDIR_] |= dborigdel
3318 v[DBDESCMAN_] += u'Delete in origin.'
3319 if rddb:
3320 v[DBDIR_] |= dbdbdel
3321 v[DBDESCMAN_] += u'Delete in db.'
3322 if not (rdbag or rdorig):
3323 v[DBDESCMAN_] += u'Nothing for deleting.'
3324 else:
3325
3326 p = getpathdirdb(v[DBPATH_])
3327 while p:
3328 pv = db[p]
3329 if pv[DBDIR_] & DBDIRDELETE:
3330 log(u'act(%s): Forced unchange parent "%s"' % (v[DBDBPATH_],p))
3331 pv[DBDIR_] = dbunchanged
3332 pv[DBDESCMAN] += u' Unchange (not all childeren deleted).'
3333 if cfg.VERBOSE >= verb:
3334 prt(u' unchange (not all childeren deleted) [%s]' % p, verbosity=verb)
3335 p = getpathdirdb(p)
3336
3337 if action == u'bag':
3338
3339
3340 if rdorig:
3341 v[DBDIR_] = dbbag
3342 v[DBDESCMAN_] += u'Copy to bag. '
3343 else:
3344 v[DBDIR_] = (v[DBDIR_] & DBDIRNEW) | dbunchanged
3345 v[DBDESCMAN_] += u'Nothing copy to bag. Unchange.'
3346 elif action == u'origin':
3347
3348
3349 if rdbag:
3350 v[DBDIR_] = dborig
3351 v[DBDESCMAN_] += u'Copy to origin.'
3352 else:
3353 v[DBDIR_] = (v[DBDIR_] & DBDIRNEW) | dbunchanged
3354 v[DBDESCMAN_] += u'Nothing copy to origin. Unchange.'
3355 elif action == u'skip':
3356
3357
3358 v[DBDIR_] = (v[DBDIR_] & DBDIRNEW) | dbunchanged
3359 v[DBDESCMAN_] += u'Unchanged.'
3360 elif action == u'restore':
3361
3362
3363 if v[DBDIRMAN_] != DBDIRNONE_:
3364 v[DBDIR_] = v[DBDIRMAN_]
3365 v[DBDESCMAN_] = u''
3366 else:
3367 log(u'act: ERROR action "%s"' % (action,))
3368 v[DBDESCMAN_] += u'ERROR action "%s"' % (action,)
3369
3370
3371
3372
3373
3374
3375
3376
3377
3378 r = u''
3379 if not (action in O_ACTION):
3380 r = u'Unknown action "%s".' % action
3381 return r
3382 actionts = []
3383
3384 if _filter:
3385 cire = _filter
3386 else:
3387 cire = u'!.*'
3388 if cire[0] == u'!':
3389 cire = cire[1:]
3390 else:
3391
3392 cire = cire.replace(u'\\',u'\\\\').replace(u'|',u'\\|').replace(u'.',u'\\.').replace(u'*',u'.*'). \
3393 replace(u'?',u'.?').replace(u'$',u'\\$').replace(u'^',u'\\^').replace(u'{',u'\\{'). \
3394 replace(u'(',u'\\(').replace(u'[',u'\\[').replace(u'+',u'\\+').split(u',')
3395 cire = u'|'.join([u'('+i+u'$)' for i in cire if i])
3396 if not cire:
3397 cire = u'.*'
3398 try:
3399 log(u'do_action: cire = "%s"' % z_u(cire))
3400 ci = re.compile(cire, re.LOCALE | re.UNICODE)
3401 except Exception, exc:
3402 prt0(u'Error when compiling _filter string "%s".' % cire, error=exc, place=u'do_action')
3403 r = u'Error when compiling _filter string.'
3404 return r
3405 dl = []
3406 if doout:
3407 dbg(u'do_action: for start.')
3408 for idb in ks:
3409 vdb = db[idb]
3410 path = vdb[DBDBPATH_]
3411 if doout:
3412 dbg(u'\ndo_action: for(path) = "%s".' % z_u(path))
3413 dbg(u'do_action: loop for dli start.')
3414
3415
3416 kk = True
3417 for dli in dl[:]:
3418 if doout:
3419 dbg(u'do_action: dli = "%s".' % dli)
3420 if dbstartswith_(idb, dli):
3421 act(vdb)
3422 kk = False
3423 if doout:
3424 dbg(u'do_action: act ok.')
3425 else:
3426 if doout:
3427 dbg(u'do_action: remove dli from dl = "%s"' % z_u(dli))
3428 dl.remove(dli)
3429 dbg(u'do_action: loop for dli end.')
3430
3431
3432 if kk and ci.match(path):
3433 dbg(u'do_action: ci.match.')
3434 act(vdb)
3435 if (vdb[DBDBFLAG_]==DBFLAGDIR) or (vdb[DBBAGFLAG_]==DBFLAGDIR) or (vdb[DBORIGFLAG_]==DBFLAGDIR):
3436
3437 if doout:
3438 dbg(u'do_action: add idb to dl = "%s"' % z_u(idb))
3439 dl.append(idb)
3440
3441 if actionts:
3442 break
3443
3444 tm = time.time() - tm
3445 prt4(u'\ndo action time is %.3f' % tm)
3446 log(u'do_action: end')
3447
3448 if actionts:
3449 return u'needcompare'
3450
3452 """
3453 Synchronize files.
3454
3455 Algorithm:
3456 - Test for attributes.
3457 - Append paths to lists.
3458 - dbread.
3459 - Loop for all paths:
3460 - dbwalkfiles.
3461 - dbcomparefiles.
3462 - dbsyncfiles.
3463 - dbcleardb
3464 - Print results.
3465 - Save DB.
3466 """
3467 log(u'start cmd_sync')
3468 z_r.clear()
3469
3470
3471
3472 prt1()
3473 t0 = time.time()
3474 _dir = DBFORCEDIRNONE
3475
3476 if cmd.d:
3477 for d in cmd.d:
3478 if Z_DBFORCE.has_key(d):
3479 _dir |= Z_DBFORCE[d]
3480 prt1(u'<y>Force direction is "</y>', d, u'"')
3481
3482 if cmd.s:
3483 try:
3484 md = float(cmd.s[-1])
3485 cfg.MDDELTA = md
3486 prt5(u'MDDELTA set to %F.' % md)
3487 except Exception, exc:
3488 prt1(u'<R>Can not set mddelta.</R>', error=exc)
3489 prt1(u'MDDELTA is %F.' % cfg.MDDELTA)
3490
3491 r = (cmd.g and cmd.g[0]) or REPORTFILTER_DEFAULT
3492 prt1(u'OUTFILTER set to "%s"' % r)
3493 r = transreport(r)
3494 if r:
3495 prt1(u'\n<Y>Wrong output filter argument. Force to "%s"</Y>\n' % REPORTFILTER_DEFAULT)
3496 r = transreport(REPORTFILTER_DEFAULT)
3497 if r:
3498 prt0(u'<R>Error in outfilter: %s</R>' % r)
3499
3500 b = (cmd.b and (cmd.b[0] in [P_BACKUPNO,P_BACKUPYES]) and cmd.b[0]) or P_BACKUPNO
3501 cfg.BACKUPMODE = bool(int(b))
3502 dbg(u'cmd_sync: BACKUPMODE = %s' % cfg.BACKUPMODE)
3503 if cmd.b:
3504 prt1(u'Backup mode is %s.' % b)
3505
3506 tsfilter = None
3507 if cmd.y:
3508 tso = cmd.y[-1]
3509 tspatt = ur"^(?P<DIR>b|o)(#(?P<FILTER>.+)|((?P<BAG>[+-]?((\d+)|(\d*\.\d+)))#(?P<ORIG>[+-]?((\d+)|(\d*\.\d+)))))$"
3510 tsre = re.compile(tspatt, re.LOCALE | re.UNICODE)
3511 m = tsre.match(tso)
3512 if m is None:
3513 prt0(u'Error in time shift option.')
3514 return
3515 m = m.groupdict()
3516 e = u'Time shift to '
3517 if m[u'DIR'] == u'b':
3518 cfg.TSDIR = TSDIRBAG
3519 e += u'bag, '
3520 else:
3521 cfg.TSDIR = TSDIRORIG
3522 e += u'origin, '
3523 tsfilter = m[u'FILTER']
3524 if tsfilter:
3525 cire = tsfilter
3526 if cire[0] == u'!':
3527 cire = cire[1:]
3528 else:
3529
3530 cire = cire.replace(u'\\',u'\\\\').replace(u'|',u'\\|').replace(u'.',u'\\.').replace(u'*',u'.*'). \
3531 replace(u'?',u'.?').replace(u'$',u'\\$').replace(u'^',u'\\^').replace(u'{',u'\\{'). \
3532 replace(u'(',u'\\(').replace(u'[',u'\\[').replace(u'+',u'\\+').split(u',')
3533 cire = u'|'.join([u'('+i+u'$)' for i in cire if i])
3534 if not cire:
3535 cire = u'.*'
3536 try:
3537 log(u'cmd_sync: cire = "%s"' % z_u(cire))
3538 ci = re.compile(cire, re.LOCALE | re.UNICODE)
3539 except Exception, exc:
3540 prt0(u'<R>Error when compiling filter string "%s".</R>' % cire, error=exc, place=u'cmd_sync')
3541 return
3542 e += u'use filter "%s".' % tsfilter
3543 tsfilterre = ci
3544 else:
3545 cfg.TSBAG = float(m[u'BAG'])
3546 cfg.TSORIG = float(m[u'ORIG'])
3547 e += u'shift bag/origin = %s/%s' % (str(cfg.TSBAG),str(cfg.TSORIG))
3548 prt0(e)
3549
3550 prt1()
3551 prt1(u'<c>Reading ...</c>\n')
3552 cfg.LOCK[LI_PERCENTPART] = 40
3553 cfg.LOCK[LI_PERCENTGLOBAL] = 0
3554 cfg.LOCK[LI_CANCEL] = False
3555 ign = f_syncread(cmd)
3556 prt4( u'All items = %d' % len(cfg.Z_DB))
3557 prt4(u' Ignored %d items in bag and origin.\n' % (ign,))
3558 if z_r.e:
3559 prt0( u'<r>ERROR occur:</r>', error=z_r.e)
3560 z_r.clear()
3561 t1 = time.time()
3562 cfg.LOCK[LI_PERCENTPART] = 10
3563 cfg.LOCK[LI_PERCENTGLOBAL] = 40
3564
3565 rd = True
3566 if tsfilter:
3567
3568 prt0(u'<c>Searching time shift template...</c>')
3569 for sk in cfg.Z_KS:
3570 vdb = cfg.Z_DB[sk]
3571
3572 rd = vdb[DBBAGREAD] and vdb[DBDBREAD] and vdb[DBORIGREAD] and (vdb[DBDBFLAG]==DBFLAGFILE) and tsfilterre.match(vdb[DBDBPATH])
3573 mdb = vdb[DBDBMDATE] - vdb[DBBAGMDATE]
3574 mdo = vdb[DBDBMDATE] - vdb[DBORIGMDATE]
3575 if abs(mdb) <= cfg.MDDELTA:
3576 mdb = 0.0
3577 if abs(mdo) <= cfg.MDDELTA:
3578 mdo = 0.0
3579 if rd:
3580 prt0(u'Time shift template used "%s".' % vdb[DBDBPATH])
3581 if mdb:
3582 cfg.TSBAG = mdb
3583 cfg.TSORIG = 0.0
3584 cfg.TSDIR = TSDIRBAG
3585 elif mdo:
3586 cfg.TSORIG = mdo
3587 cfg.TSBAG = 0.0
3588 cfg.TSDIR = TSDIRBAG
3589 else:
3590
3591 cfg.TSBAG = 0.0
3592 cfg.TSORIG = 0.0
3593 cfg.TSDIR = TSDIRNONE
3594 prt0(u'<y>Set time shift to %s, shift bag/origin = %s/%s</y>' % ([u'None',u'Bag',u'Origin'][cfg.TSDIR], str(cfg.TSBAG), str(cfg.TSORIG)))
3595 break
3596 if not rd:
3597 prt0(u'<R>Time shift template not found. Exit.</R>')
3598 return
3599
3600
3601
3602 repeat = True
3603 needcompare = True
3604 needreport = cfg.REPORTSHOW
3605 while repeat:
3606 log(u'cmd_sync: Menu loop repeat.')
3607 if needcompare:
3608 t12 = time.time()
3609 prt1()
3610 prt1(u'<c>Comparing ...</c>')
3611
3612 prt1()
3613 ren,rc,rw,rbn,ron,rbc,roc,rbd,rod, rign, rts = f_synccompare(_dir)
3614 t2 = time.time()
3615 prt1()
3616 z_r.clear()
3617 needcompare = False
3618
3619
3620 if cmd.p:
3621 prt1()
3622 prt1( u'\t errors = <r>%d</r>' % (ren,))
3623 prt1( u'\t conflicts = <Y>%d</Y>' % (rc,))
3624 prt1( u'\t warnings = <y>%d</y>' % (rw,))
3625 prt1( u'\t all items = %d' % len(cfg.Z_DB))
3626 prt1( u'\t new in bag = %d' % (rbn,))
3627 prt1( u'\t new in origin = %d' % (ron,))
3628 prt1( u'\t changed in bag = %d' % (rbc,))
3629 prt1( u'\t changed in origin = %d' % (roc,))
3630 prt1( u'\t deleted in bag = %d' % (rbd,))
3631 prt1( u'\t deleted in origin = %d' % (rod,))
3632 prt1( u'\t ignored = %d' % (rign,))
3633 prt1( u'\t timeshifted = %d' % (rts,))
3634 prt1()
3635 if cmd.p[-1] != u'y':
3636 prt1(u'\n<c>Synchronization canceled.</c>\n')
3637 prt1()
3638 prt1( u'Reading duration is %.3f sec.' % (t1 - t0,) )
3639 prt1( u'Comparing duration is %.3f sec.' % (t2 - t1,) )
3640 repeat = False
3641 log(u'end cmd_sync')
3642 return
3643 else:
3644
3645 repeat = False
3646 repeatmenu = False
3647 needreport = False
3648 break
3649
3650 if needreport:
3651 outreport()
3652 needreport = False
3653
3654 prt1()
3655 prt1( u'\t errors = <r>%d</r>' % (ren,))
3656 prt1( u'\t conflicts = <Y>%d</Y>' % (rc,))
3657 prt1( u'\t warnings = <y>%d</y>' % (rw,))
3658 prt1( u'\t all items = %d' % len(cfg.Z_DB))
3659 prt1( u'\t new in bag = %d' % (rbn,))
3660 prt1( u'\t new in origin = %d' % (ron,))
3661 prt1( u'\t changed in bag = %d' % (rbc,))
3662 prt1( u'\t changed in origin = %d' % (roc,))
3663 prt1( u'\t deleted in bag = %d' % (rbd,))
3664 prt1( u'\t deleted in origin = %d' % (rod,))
3665 prt1( u'\t ignored = %d' % (rign,))
3666 prt1( u'\t timeshifted = %d' % (rts,))
3667 prt1()
3668
3669 repeatmenu = True
3670
3671 while repeatmenu:
3672 log(u'cmd_sync: Menu loop repeatmenu.')
3673
3674 prt(climenu, verbose=-100)
3675
3676
3677
3678
3679
3680
3681
3682
3683 try:
3684 b = raw_input()
3685 prt(u'', verbose=-100)
3686 b = z_u(b.strip(),cfg.Z_INEP)
3687 log(u'cmd_sync.menu input: "%s"' % b)
3688
3689 a = [i for i in b.split(u' ') if i]
3690 if a[0] == u'y':
3691 prt(u'\n<c>Synchronization continued.</c>\n', verbose=-100)
3692 repeat = False
3693 repeatmenu = False
3694 elif a[0] == u'n':
3695 prt(u'\n<c>Synchronization canceled.</c>\n', verbose=-100)
3696 prt1()
3697 prt1( u'Reading duration is %.3f sec.' % (t1 - t0,) )
3698 prt1( u'Comparing duration is %.3f sec.' % (t2 - t12,) )
3699 prt5(u'end cmd_sync')
3700 repeat = False
3701 repeatmenu = False
3702 return
3703 elif a[0] == u'h':
3704 prt(climenuhelp, verbose=-100)
3705 elif a[0] == u'd':
3706
3707 if len(a) < 2:
3708 prt(u'\n<r>Wrong number arguments.</r>\n', verbose=-100)
3709 continue
3710 _dir = DBDIRNONE
3711 for i in a[1:]:
3712 if Z_DBFORCE.has_key(i):
3713 _dir |= Z_DBFORCE[i]
3714 prt(u'\nDirection rule "%s" will be used.\n' % (i,), verbose=-100)
3715 else:
3716 prt(u'\n<r>Wrong direction "%s". Skiped.</r>\n' % (i,), verbose=-100)
3717 needcompare = True
3718 repeatmenu = False
3719 needreport = True
3720 elif a[0] == u'm':
3721
3722 if len(a) != 2:
3723 prt(u'\n<r>Wrong number arguments.</r>\n', verbose=-100)
3724 continue
3725 if a[1] and (a[1] in [u'0', u'1']):
3726 cfg.PYBAG_EMUL = bool(int(a[1]))
3727 prt(u'\nEmulation level changed to "%d".\n' % (cfg.PYBAG_EMUL,), verbose=-100)
3728 else:
3729 prt(u'\n<r>Wrong emulation level "%s".</r>\n' % (a[1],), verbose=-100)
3730 elif a[0] == u't':
3731
3732 if len(a) != 2:
3733 prt(u'\n<r>Wrong number arguments.</r>\n', verbose=-100)
3734 continue
3735 if a[1] and (a[1] in [u'0', u'1']):
3736 cfg.PYBAG_DEBUG = bool(int(a[1]))
3737 prt(u'\nDebug level changed to "%d".\n' % (cfg.PYBAG_DEBUG,), verbose=-100)
3738 else:
3739 prt(u'\n<r>Wrong debug level "%s".</r>\n' % (a[1],), verbose=-100)
3740 elif a[0] == u'v':
3741
3742 if len(a) != 2:
3743 prt(u'\n<r>Wrong number arguments.</r>\n', verbose=-100)
3744 continue
3745 if a[1] and (a[1] in [u'-1', u'0', u'1', u'2', u'3', u'4', u'5']):
3746 cfg.VERBOSE = int(a[1])
3747 prt(u'\nVerbosity level changed to "%d".\n' % (cfg.VERBOSE,), verbose=-100)
3748 else:
3749 prt(u'\n<r>Wrong verbosity level "%s".</r>\n' % (a[1],), verbose=-100)
3750 elif a[0] == u'a':
3751
3752 if len(a) < 2:
3753 prt(u'\n<r>Wrong number arguments.</r>\n', verbose=-100)
3754 continue
3755 inter = False
3756 listp = True
3757 if a[1][0] == u'-':
3758 listp = False
3759 a[1] = a[1][1:]
3760 if a[1][0] == u'+':
3761 inter = True
3762 a[1] = a[1][1:]
3763 if len(a) == 2:
3764 dbg(u'cmd_sync: action="%s"' % a[1])
3765 r = do_action( a[1], u'!.*', inter, listp )
3766 else:
3767 c = 0
3768 b = b.split()
3769 while c < 2 and b:
3770 if b[0].strip():
3771 c += 1
3772 del b[0]
3773 b = u' '.join(b)
3774 dbg(u'cmd_sync: action="%s", filter="%s"' % (a[1],b))
3775 r = do_action( a[1], b , inter, listp)
3776 if r == u'needcompare':
3777 needcompare = True
3778 repeatmenu = False
3779 needreport = True
3780 elif r:
3781 prt(u'\n%s' % r, verbose=-100)
3782 continue
3783 elif a[0] == u'g':
3784
3785 r = transreport( (len(a)>=2 and b[1:].strip()) or REPORTFILTER_DEFAULT )
3786 if r:
3787 prt(u'\n<r>Wrong output filter argument: "%s"</r>.\n' % r, verbose=-100)
3788 continue
3789 needreport = True
3790 repeatmenu = False
3791 elif a[0] == u'r' and len(a) == 1:
3792
3793 needreport = True
3794 repeatmenu = False
3795 else:
3796 prt(u'\n<r>Wrong command "%s".</r>\n' % (a[0],), verbose=-100)
3797
3798 except Exception, exc:
3799 prt0(u'\n<r>Error then interpret answer. Try again.</r>\n', place=u'cmd_sync', error=exc)
3800
3801
3802
3803
3804
3805 prt1(u'<c>Synchronizing ...</c>\n')
3806 cfg.LOCK[LI_PERCENTPART] = 40
3807 cfg.LOCK[LI_PERCENTGLOBAL] = 50
3808 t4 = time.time()
3809
3810 sd, sn, sc, su, se, ts = f_syncsync()
3811 prt1()
3812 prt1( u'\t errors after comparing = <r>%d</r>' % (su,))
3813 prt1( u'\t errors in synchronizing = <r>%d</r>' % (se,))
3814 prt1( u'\t all items = %d' % len(cfg.Z_DB))
3815 prt1( u'\t new = %d' % (sn,))
3816 prt1( u'\t copied = %d' % (sc,))
3817 prt1( u'\t deleted = %d' % (sd,))
3818 prt1( u'\t timeshifted = %d' % (ts,))
3819 z_r.clear()
3820 t5 = time.time()
3821 prt1()
3822 prt1(u'<c>Clearing ...</c>\n')
3823 cfg.LOCK[LI_PERCENTPART] = 10
3824 cfg.LOCK[LI_PERCENTGLOBAL] = 90
3825 f_syncclear()
3826 t6 = time.time()
3827 prt1()
3828 prt1(u'Save DB ...\n')
3829 dbsave(cfg.Z_DB)
3830 t7 = time.time()
3831 prt1()
3832 prt1(u'<c>Synchronization complete.</c>')
3833 prt1()
3834 prt1( u'Reading duration is %.3f sec.' % (t1 - t0,) )
3835 prt1( u'Comparing duration is %.3f sec.' % (t2 - t12,) )
3836 prt1( u'Synchronization duration is %.3f sec.' % (t5 - t4,) )
3837 prt1( u'Clearing duration is %.3f sec.' % (t6 - t5,) )
3838 prt1( u'Save duration is %.3f sec.' % (t7 - t6,) )
3839 prt1()
3840 tt = t7 - t4 + t1 - t0 + t2 - t12
3841 prt(u'Summary synchronization time is %.3f sec.' % (tt,))
3842 prt1()
3843
3844 log(u'end cmd_sync')
3845
3910 z_r.l = ret
3911
3912
3913 LOCK_[LI_DESC_] = u'Reading DB.'
3914 LOCK_[LI_PERCENT100_] += len(_db_) * 4 + 2
3915 LOCK_[LI_PERCENT_] = 0
3916 if LOCK_[LI_CANCEL_]:
3917 log_(u'f_syncread: canceled.')
3918 return ign
3919
3920 mp = None
3921 vf = []
3922 if cmd.fs and cmd.fs[-1]:
3923 mp = cmd.fs[-1]
3924 dbg_(u'f_syncread: mp="%s".' % (mp,))
3925
3926 if cmd.f and len(cmd.f) == 1:
3927
3928
3929 dbg_(u'f_syncread: f[0]="%s".' % (cmd.f[0],))
3930 rn = cfg.Z_CP.findroot(cmd.f[0])
3931 if rn:
3932 dbg_(u'f_syncread: rn="%s".' % (rn,))
3933 r = validatepaths(rn, cmd.f[0], mp)
3934 vf.append(r)
3935 else:
3936 prt0(u'<r>Root "%s" in pybagfiles not found.</r>' % (cmd.f[0],), place=u'f_syncread')
3937 else:
3938 if cmd.f:
3939
3940 dbg_(u'f_syncread: Get specified roots.')
3941 for p in cmd.f:
3942 rn = cfg.Z_CP.findroot(p)
3943 if rn:
3944 dbg(u'f_syncread: root="%s".' % (rn,))
3945 sp = None
3946 if mp:
3947
3948 sp = os.path.join(mp, p)
3949 dbg(u'f_syncread: sp="%s".' % (sp,))
3950 r = validatepaths(rn, p, sp)
3951 vf.append(r)
3952 else:
3953 prt0(u'<r>Root "%s" in pybagfiles not found.</r>' % (cmd.f[0],), place=u'f_syncread')
3954 else:
3955 rs = []
3956 if cmd.x:
3957
3958 if cmd.x[0] == u'default':
3959
3960 rbs = cfg.Z_CP.getroots()
3961 prt1()
3962 for rn in rbs:
3963 rbn = cfg.Z_CP.getrootbag(rn)
3964 seldef = cfg.Z_CP.getopt(rn, P_SELECTDEFAULT, False, P_SELECTDEFAULTNO)
3965 seldef = seldef==P_SELECTDEFAULTYES
3966 if seldef:
3967 rs.append(rn)
3968 prt1(u'\tRoot "%s" marked for synchronization.' % rbn)
3969 prt1()
3970 elif cmd.x[0] == u'all':
3971 rs = cfg.Z_CP.getroots()
3972 else:
3973 prt1(u'\n<r>Option x is wrong. Do not add roots to sync.</r>',place=u'f_dist')
3974 else:
3975
3976 prt1(u'Get default roots for synchronization...')
3977 rs = []
3978
3979 rbs = cfg.Z_CP.getroots()
3980 prt1()
3981 for rn in rbs:
3982 rbn = cfg.Z_CP.getrootbag(rn)
3983 seldef = cfg.Z_CP.getopt(rn, P_SELECTDEFAULT, False, P_SELECTDEFAULTNO)
3984 seldef = seldef==P_SELECTDEFAULTYES
3985 if seldef:
3986 rs.append(rn)
3987 prt1(u'\tRoot "%s" marked for synchronization.' % rbn)
3988 prt1()
3989
3990
3991 log(u'roots=%s' %(rs,))
3992
3993 for rn in rs:
3994 dbg(u'f_syncread: root="%s".' % (rn,))
3995 r = validatepaths(rn, None, None)
3996 vf.append(r)
3997 vf = [p for p in vf if p]
3998 log(u'f_syncread: Number of roots for sync=%d.' % (len(vf,)))
3999 log(u'f_syncread: vf=%s' % str(vf))
4000
4001 LOCK_[LI_PERCENT_] += 1
4002 if LOCK_[LI_CANCEL_]:
4003 log_(u'f_syncread: canceled.')
4004 return ign
4005
4006
4007 log(u'f_syncread: Walk.')
4008 for r in vf:
4009
4010 back = cfg.Z_CP.getopt(r[0], P_BACKUP, P_BACKUPNO) == P_BACKUPYES
4011 dbg_(u'f_syncread: back = %s for "%s"' % (back,r[1]))
4012 if not back or BACKUPMODE_:
4013
4014 if doout:
4015 prt4_(u'WALK for\n\t"%s" and "%s".\n' % (r[1],r[2]))
4016 (_s, _i) = dbwalkfiles(cfg.Z_DB, r[0], r[1], True, r[3])
4017 ign += _i
4018
4019 r.append(back)
4020
4021 cs = cfg.Z_CP.getopt(r[0], P_SYMLINK, False, P_DEFSYMLINK)
4022 cse = cs in P_SYMLINK_EMUL
4023 dbg_(u'f_syncread: cse = %s for "%s"' % (cse,r[1]))
4024 r.append(cse)
4025
4026 if LOCK_[LI_CANCEL_]:
4027 log_(u'f_syncread: canceled.')
4028 return ign
4029 (_s, _i) = dbwalkfiles(cfg.Z_DB, r[0], r[2], False, r[3])
4030 ign += _i
4031 if LOCK_[LI_CANCEL_]:
4032 log(u'f_syncread: canceled.')
4033 return ign
4034
4035 LOCK_[LI_PERCENT100_] = LOCK_[LI_PERCENT_] + len(_db_) * 2 + 2
4036
4037
4038 dbnormalize(cfg.Z_DB)
4039
4040 if LOCK_[LI_CANCEL_]:
4041 log_(u'f_syncread: canceled.')
4042 return ign
4043
4044
4045 log_(u'f_syncread: Fill paths.')
4046 lenvf = len(vf)
4047 for k, vdb in _db_.iteritems():
4048
4049 if doout:
4050 dbg_(u'f_syncread: \tFill paths for k="%s".' % k)
4051
4052 vdb[DBPATH_] = k
4053 vdb[DBDBPATH_] = path2os_(k)
4054 try:
4055 fl = False
4056 for r in vf:
4057
4058
4059 if dbstartswith_(k,r[3]):
4060
4061 vdb[DBBACKUP_] = r[4]
4062 if doout:
4063 dbg_(u'f_syncread: r[4]back =%s.' % r[4])
4064
4065 vdb[DBSYMLINKEMUL_] = r[5]
4066 if r[4] and not BACKUPMODE_:
4067
4068 vdb[DBBAGFLAG_] = vdb[DBDBFLAG_]
4069 vdb[DBBAGLINK_] = vdb[DBDBLINK_]
4070 vdb[DBBAGMDATE_] = vdb[DBDBMDATE_]
4071
4072 vdb[DBBAGREAD_] = vdb[DBDBREAD_]
4073 vdb[DBBAGSIZE_] = vdb[DBDBSIZE_]
4074 if doout:
4075 dbg_(u'f_syncread: backup copy to bag.')
4076 vdb[DBSYNC_] |= DBSYNCCOMPARE_
4077
4078 fl = True
4079 pe = k[len(r[3]):]
4080
4081
4082 if pe.startswith(DBSEP_):
4083 pe = pe[1:]
4084
4085
4086 pb = r[1]
4087 po = r[2]
4088 if pe:
4089 pe = path2os_(pe)
4090 pb = os_.path.join(pb,pe)
4091 po = os_.path.join(po,pe)
4092
4093
4094
4095
4096
4097
4098 vdb[DBBAGPATH_] = pb
4099
4100
4101
4102
4103 vdb[DBORIGPATH_] = po
4104
4105
4106 continue
4107
4108
4109 if not fl:
4110 log_(u'f_syncread: \tDo not compare k="%s".' % (k,))
4111 LOCK_[LI_PERCENT_] += lenvf
4112 if LOCK_[LI_CANCEL_]:
4113 log_(u'f_syncread: canceled.')
4114 return ign
4115 except Exception, exc:
4116 prt5_(u'<R>f_syncread: ERROR: k="%s". %s</R>' % (k,_u_(exc)))
4117 raise exc
4118
4119
4120 cfg.Z_KS = _db_.keys()
4121 cfg.Z_KS.sort(reverse=False)
4122
4123 log_(u'end f_syncread')
4124 return ign
4125
4127 """
4128 Compare DB.
4129
4130 Return tuple of number modified items.
4131
4132 @warning: All paths (BAG, Origin) must be filled. And DB must be normalized.
4133
4134 @param direction: Forced direction in conflicts.
4135 @type direction: int { DBFORCEDIR*** constants }
4136 """
4137
4138 _db_ = cfg.Z_DB
4139 LOCK_ = cfg.LOCK
4140 LI_PERCENT100_ = LI_PERCENT100
4141 LI_DESC_ = LI_DESC
4142 LI_PERCENT_ = LI_PERCENT
4143 LI_CANCEL_ = LI_CANCEL
4144 dbg_ = dbg
4145 path2os_ = path2os
4146 os_path_lexists_ = os.path.lexists
4147 rmdirr_ = rmdirr
4148 shutil_copy2_ = shutil.copy2
4149 getcrc32_ = getcrc32
4150 readlink_ = readlink
4151 os_remove_ = os.remove
4152 symlink_ = symlink
4153 os_makedirs_ = os.makedirs
4154 os_path_dirname_ = os.path.dirname
4155 prt1_ = prt1
4156 prt2_ = prt2
4157 prt3_ = prt3
4158 prt4_ = prt4
4159 prt5_ = prt5
4160 doout = (cfg.VERBOSE >= 4) or cfg.PYBAG_DEBUG
4161 log_ = log
4162 PYBAG_EMUL_ = cfg.PYBAG_EMUL
4163 DBBAGFLAG_ = DBBAGFLAG
4164 DBBAGSIZE_ = DBBAGSIZE
4165 DBBAGMDATE_ = DBBAGMDATE
4166 DBBAGCRC32_ = DBBAGCRC32
4167 DBBAGREAD_ = DBBAGREAD
4168 DBBAGPATH_ = DBBAGPATH
4169 DBBAGLINK_ = DBBAGLINK
4170 DBDBFLAG_ = DBDBFLAG
4171 DBDBSIZE_ = DBDBSIZE
4172 DBDBMDATE_ = DBDBMDATE
4173 DBDBCRC32_ = DBDBCRC32
4174 DBDBREAD_ = DBDBREAD
4175 DBDBPATH_ = DBDBPATH
4176 DBDBLINK_ = DBDBLINK
4177 DBORIGFLAG_ = DBORIGFLAG
4178 DBORIGSIZE_ = DBORIGSIZE
4179 DBORIGMDATE_ = DBORIGMDATE
4180 DBORIGCRC32_ = DBORIGCRC32
4181 DBORIGREAD_ = DBORIGREAD
4182 DBORIGPATH_ = DBORIGPATH
4183 DBORIGLINK_ = DBORIGLINK
4184 DBDIR_ = DBDIR
4185 DBSYNC_ = DBSYNC
4186 DBCOMPARE_ = DBCOMPARE
4187 DBSYMLINK_ = DBSYMLINK
4188 DBDESC_ = DBDESC
4189 DBDIRMAN_ = DBDIRMAN
4190 DBDESCMAN_ = DBDESCMAN
4191 DBITEMID_ = DBITEMID
4192 DBDIRREADED_ = DBDIRREADED
4193 DBDESCREADED_ = DBDESCREADED
4194 DBREADYES_ = DBREADYES
4195 DBREADNONE_ = DBREADNONE
4196 DBFLAGNONE_ = DBFLAGNONE
4197 DBFLAGDIR_ = DBFLAGDIR
4198 DBFLAGFILE_ = DBFLAGFILE
4199 DBFLAGSYMLINK_ = DBFLAGSYMLINK
4200 DBFLAGUNKNOWN_ = DBFLAGUNKNOWN
4201 DBMDATENONE_ = DBMDATENONE
4202 DBNONE_ = DBNONE
4203 DBPATHNONE_ = DBPATHNONE
4204 DBDESCNONE_ = DBDESCNONE
4205 DBDESCMANNONE_ = DBDESCMANNONE
4206 DBDESCOK_ = DBDESCOK
4207 DBLINKNONE_ = DBLINKNONE
4208 DBITEMNONE_ = DBITEMNONE
4209 DBDIRNONE_ = DBDIRNONE
4210 DBDIRBAG_ = DBDIRBAG
4211 DBDIRORIG_ = DBDIRORIG
4212 DBDIRDB_ = DBDIRDB
4213 DBDIRCOPY_ = DBDIRCOPY
4214 DBDIRDELETE_ = DBDIRDELETE
4215 DBDIRWARNING_ = DBDIRWARNING
4216 DBDIRCONFLICT_ = DBDIRCONFLICT
4217 DBDIRERROR_ = DBDIRERROR
4218 DBDIRNEW_ = DBDIRNEW
4219 DBDIRUNCHANGED_ = DBDIRUNCHANGED
4220 DBDIRFORCED_ = DBDIRFORCED
4221 DBDIRIGNORED_ = DBDIRIGNORED
4222 DBDIRDIFF_ = DBDIRDIFF
4223 DBDIROK_ = DBDIROK
4224 DBDIRRESTORE_ = DBDIRRESTORE
4225 DBSYNCNONE_ = DBSYNCNONE
4226 DBSYNCERRREADBAG_ = DBSYNCERRREADBAG
4227 DBSYNCERRREADSYNC_ = DBSYNCERRREADSYNC
4228 DBSYNCERRREAD_ = DBSYNCERRREAD
4229 DBSYNCERRWRITEBAG_ = DBSYNCERRWRITEBAG
4230 DBSYNCERRWRITESYNC_ = DBSYNCERRWRITESYNC
4231 DBSYNCERRWRITE_ = DBSYNCERRWRITE
4232 DBSYNCOK_ = DBSYNCOK
4233 DBSYNCERROTHER_ = DBSYNCERROTHER
4234 DBSYNCERRCOMPARE_ = DBSYNCERRCOMPARE
4235 DBSYNCERROR_ = DBSYNCERROR
4236 DBSYNCCOMPARE_ = DBSYNCCOMPARE
4237 DBSYNCIGNORED_ = DBSYNCIGNORED
4238 DBSYNCCONFLICT_ = DBSYNCCONFLICT
4239 DBFORCEDIRNONE_ = DBFORCEDIRNONE
4240 DBFORCEDIRBAG_ = DBFORCEDIRBAG
4241 DBFORCEDIRORIGIN_ = DBFORCEDIRORIGIN
4242 DBFORCEDIROLDER_ = DBFORCEDIROLDER
4243 DBFORCEDIRNEWER_ = DBFORCEDIRNEWER
4244 DBFORCEDIRBAGALL_ = DBFORCEDIRBAGALL
4245 DBFORCEDIRORIGINALL_ = DBFORCEDIRORIGINALL
4246 DBFORCEDIRORIGINCOPY_ = DBFORCEDIRORIGINCOPY
4247 DBFORCEDIRBAGCOPY_ = DBFORCEDIRBAGCOPY
4248 P_COMPARECRC32_ = P_COMPARECRC32
4249 DBBACKUP_ = DBBACKUP
4250 BACKUPMODE_ = cfg.BACKUPMODE
4251 BACKUPDIR_ = BACKUPDIR
4252
4253 MDDELTA_ = cfg.MDDELTA
4254 DBREADNNN_ = (DBREADNONE_, DBREADNONE_, DBREADNONE_)
4255 DBREADNNY_ = (DBREADNONE_, DBREADNONE_, DBREADYES_)
4256 DBREADNYN_ = (DBREADNONE_, DBREADYES_, DBREADNONE_)
4257 DBREADNYY_ = (DBREADNONE_, DBREADYES_, DBREADYES_)
4258 DBREADYNN_ = (DBREADYES_, DBREADNONE_, DBREADNONE_)
4259 DBREADYNY_ = (DBREADYES_, DBREADNONE_, DBREADYES_)
4260 DBREADYYN_ = (DBREADYES_, DBREADYES_, DBREADNONE_)
4261 DBREADYYY_ = (DBREADYES_, DBREADYES_, DBREADYES_)
4262
4263 DBDIRCOPY_ORIG_DB_ = DBDIRCOPY_ | DBDIRORIG_ | DBDIRDB_
4264 DBDIRCOPY_BAG_DB_ = DBDIRCOPY_ | DBDIRBAG_ | DBDIRDB_
4265 DBDIRCOPY_DB_ = DBDIRCOPY_ | DBDIRDB_
4266 DBDIRBAG_DB_NEW_ = DBDIRBAG_ | DBDIRDB_ | DBDIRNEW_
4267 DBDIRORIG_DB_NEW_ = DBDIRORIG_ | DBDIRDB_ | DBDIRNEW_
4268 DBDIRWARNING_NEW_DELETE_COPY_ = DBDIRWARNING_ | DBDIRNEW_ | DBDIRDELETE_ | DBDIRCOPY_
4269 LOOPN_ = LOOPN
4270
4271 TSDIR_ = cfg.TSDIR
4272 TSDIRBAG_ = TSDIRBAG
4273 TSDIRORIG_ = TSDIRORIG
4274 TSDIRNONE_ = TSDIRNONE
4275 DBDIRTS_ = DBDIRTS
4276 tsdirnone = TSDIR_ == TSDIRNONE_
4277 dbdirts = DBDIRUNCHANGED_ | DBDIRTS_
4278 if TSDIR_ == TSDIRBAG_:
4279 tsdesc = u'Unchanged. Time shift to bag.'
4280 else:
4281 tsdesc = u'Unchanged. Time shift to origin.'
4282
4283 log_(u'start f_synccompare')
4284 prt4_(u'\nCOMPARE FILES\n')
4285 re = 0
4286 rc = 0
4287 rw = 0
4288 rbn = 0
4289 ron = 0
4290 rbc = 0
4291 roc = 0
4292 rbd = 0
4293 rod = 0
4294 rign = 0
4295 rts = 0
4296
4297 LOCK_[LI_PERCENT100_] = len(_db_) + 1
4298 LOCK_[LI_DESC_] = u'Compare files.'
4299 LOCK_[LI_PERCENT_] = 0
4300 LN = 0
4301 if LOCK_[LI_CANCEL_]:
4302 prt3_(u'CANCELED.',place=u'f_synccompare')
4303 return (re,rc,rw,rbn,ron,rbc,roc,rbd,rod,rign)
4304
4305 for idb,vdb in _db_.iteritems():
4306 if doout:
4307 prt4_(u' ... "%s"' % vdb[DBDBPATH_], place=u'f_synccompare')
4308 LN += 1
4309 if LN > LOOPN_:
4310 LOCK_[LI_PERCENT_] += LN
4311 LN = 0
4312 if LOCK_[LI_CANCEL_]:
4313 dbg_(u'f_synccompare: canceled.')
4314 return (re,rc,rw,rbn,ron,rbc,roc,rbd,rod,rign)
4315
4316 vdb[DBDIR_] = vdb[DBDIRREADED_]
4317 vdb[DBDESC_] = vdb[DBDESCREADED_]
4318 vdb[DBDIRMAN_] = DBDIRNONE_
4319 vdb[DBDESCMAN_] = DBDESCMANNONE_
4320
4321 vdb_DBDIR_ = vdb[DBDIR_]
4322 if vdb[DBSYNC_] & DBSYNCERROR_:
4323
4324 vdb[DBDIR_] = DBDIRERROR_
4325 if vdb[DBSYNC_] & DBSYNCERRREAD_:
4326 vdb[DBDESC_] = u'Error then read from bag.'
4327 else:
4328 vdb[DBDESC_] = u'Error.'
4329 prt4_(u'\t== "%s".' % vdb[DBDESC_], place=u'f_synccompare')
4330 continue
4331 if not (vdb[DBSYNC_] & DBSYNCCOMPARE_):
4332
4333 vdb[DBDESC_] = u'Do not compare.'
4334 prt5_(u'\tDo not compare idb="%s".' % (idb))
4335 continue
4336 if (vdb_DBDIR_ & DBDIRIGNORED_):
4337
4338 vdb[DBDESC_] = u'Ignored.'
4339 rign += 1
4340 prt5_(u'\tIgnore idb="%s".' % (idb))
4341 continue
4342
4343 back = vdb[DBBACKUP_] and not BACKUPMODE_
4344 dbr = vdb[DBDBREAD_]
4345 if back:
4346
4347 bagr = dbr
4348 _dir = BACKUPDIR_
4349 else:
4350 bagr = vdb[DBBAGREAD_]
4351 _dir = direction
4352 if vdb[DBBACKUP_]:
4353 _dir = BACKUPDIR_
4354 syncr = vdb[DBORIGREAD_]
4355 rdc = (bagr, dbr, syncr)
4356 if doout:
4357 dbg_(u'f_synccompare: \tCASE(%s %s %s) back=%d, dir=%d.' % (bagr, dbr, syncr,back,_dir))
4358
4359 isfilebag = vdb[DBBAGFLAG_] == DBFLAGFILE_
4360 isfileorig = vdb[DBORIGFLAG_] == DBFLAGFILE_
4361 if isfilebag:
4362
4363 TSBAG_B = cfg.TSBAG
4364
4365 else:
4366 TSBAG_B = 0.0
4367
4368 if isfileorig:
4369
4370
4371 TSORIG_O = cfg.TSORIG
4372 else:
4373
4374 TSORIG_O = 0.0
4375
4376
4377 if rdc == DBREADYYY_ :
4378
4379
4380 vdb_crc32_ = vdb[DBCOMPARE_] == P_COMPARECRC32_ and vdb[DBBAGFLAG_] != DBFLAGSYMLINK_
4381 if not back:
4382 eq1 = (vdb[DBBAGFLAG_] == vdb[DBDBFLAG_] and
4383 vdb[DBBAGSIZE_] == vdb[DBDBSIZE_] and
4384 vdb[DBBAGLINK_] == vdb[DBDBLINK_] and
4385 (abs(vdb[DBDBMDATE_] - vdb[DBBAGMDATE_] - TSBAG_B)<=MDDELTA_) )
4386 if eq1 and vdb_crc32_:
4387
4388 vdb[DBBAGCRC32_] = tmp_ = getcrc32_(vdb[DBBAGPATH_])
4389 eq1 = eq1 and tmp_ == vdb[DBDBCRC32_]
4390 else:
4391
4392 eq1 = True
4393
4394 eq2= (vdb[DBDBFLAG_] == vdb[DBORIGFLAG_] and
4395 vdb[DBDBSIZE_] == vdb[DBORIGSIZE_] and
4396 vdb[DBDBLINK_] == vdb[DBORIGLINK_] and
4397 (abs(vdb[DBDBMDATE_] - vdb[DBORIGMDATE_] - TSORIG_O)<=MDDELTA_) )
4398 if eq2 and vdb_crc32_:
4399
4400 vdb[DBORIGCRC32_] = tmp_ = getcrc32_(vdb[DBORIGPATH_])
4401 eq2 = eq2 and vdb[DBDBCRC32_] == tmp_
4402 if eq1 and eq2:
4403
4404 if tsdirnone or not isfilebag:
4405 vdb[DBDIR_] = DBDIRUNCHANGED_
4406 vdb[DBDESC_] = u'Unchanged.'
4407 else:
4408
4409 vdb[DBDIR_] = dbdirts
4410 vdb[DBDESC_] = tsdesc
4411 rts += 1
4412 elif eq1 and not eq2:
4413
4414 vdb[DBDIR_] = DBDIRCOPY_BAG_DB_
4415 vdb[DBDESC_] = u'Changed in origin. Copy to bag.'
4416 roc += 1
4417 if _dir & DBFORCEDIRORIGINALL_ or _dir & DBFORCEDIRORIGINCOPY_:
4418
4419 vdb[DBDIR_] = DBDIRCOPY_ORIG_DB_ | DBDIRFORCED_
4420 vdb[DBDESC_] += u' !!! Force ORIGINCOPY/ALL - copy to origin.'
4421 elif not eq1 and eq2:
4422
4423 vdb[DBDIR_] = DBDIRCOPY_ORIG_DB_
4424 vdb[DBDESC_] = u'Changed in bag. Copy to origin.'
4425 rbc += 1
4426 if _dir & DBFORCEDIRBAGALL_ or _dir & DBFORCEDIRBAGCOPY_:
4427
4428 vdb[DBDIR_] = DBDIRCOPY_BAG_DB_ | DBDIRFORCED_
4429 vdb[DBDESC_] += u' !!! Force BAGCOPY/ALL - copy to bag.'
4430 else:
4431
4432 eq3= (vdb[DBBAGFLAG_] == vdb[DBORIGFLAG_] and
4433 vdb[DBBAGSIZE_] == vdb[DBORIGSIZE_] and
4434 vdb[DBBAGLINK_] == vdb[DBORIGLINK_] and
4435 (abs(vdb[DBBAGMDATE_] + TSBAG_B - vdb[DBORIGMDATE_] - TSORIG_O)<=MDDELTA_) )
4436 if eq3 and vdb_crc32_ and (vdb[DBBAGCRC32_] == vdb[DBORIGCRC32_]):
4437
4438 vdb[DBDIR_] = DBDIRCOPY_DB_
4439 vdb[DBDESC_] = u'Warning. Manualy changed in bag and origin. Copy to DB.'
4440 roc += 1
4441 rbc += 1
4442 rw += 1
4443 else:
4444
4445
4446 vdb[DBDIR_] = DBDIRCONFLICT_
4447 vdb[DBDESC_] = u'Conflict. Item in bag and origin is changed.'
4448 rc += 1
4449 dd = _dir
4450 if dd & DBFORCEDIRNEWER_:
4451
4452 if vdb[DBBAGMDATE_] + TSBAG_B - vdb[DBORIGMDATE_] - TSORIG_O < -MDDELTA_:
4453 dd = DBFORCEDIRBAG_
4454 vdb[DBDESC_] += u' Origin is newer.'
4455 elif vdb[DBBAGMDATE_] + TSBAG_B - vdb[DBORIGMDATE_] - TSORIG_O > MDDELTA_:
4456 dd = DBFORCEDIRORIGIN_
4457 vdb[DBDESC_] += u' Bag is newer.'
4458 if dd & DBFORCEDIROLDER_:
4459
4460 if vdb[DBBAGMDATE_] + TSBAG_B - vdb[DBORIGMDATE_] - TSORIG_O < -MDDELTA_:
4461 dd = DBFORCEDIRORIGIN_
4462 vdb[DBDESC_] += u' Bag is older.'
4463 elif vdb[DBBAGMDATE_] + TSBAG_B - vdb[DBORIGMDATE_] - TSORIG_O > MDDELTA_:
4464 dd = DBFORCEDIRBAG_
4465 vdb[DBDESC_] += u' Origin is older.'
4466 if dd & DBFORCEDIRBAGALL_ or dd & DBFORCEDIRBAGCOPY_:
4467
4468 vdb[DBDIR_] = DBDIRCOPY_BAG_DB_ | DBDIRFORCED_
4469 vdb[DBDESC_] += u' !!! Force BAGCOPY/ALL - copy to bag.'
4470 if _dir & DBFORCEDIRORIGINALL_ or _dir & DBFORCEDIRORIGINCOPY_:
4471
4472 vdb[DBDIR_] = DBDIRCOPY_ORIG_DB_ | DBDIRFORCED_
4473 vdb[DBDESC_] += u' !!! Force ORIGINCOPY/ALL - copy to origin.'
4474 if dd & DBFORCEDIRBAG_:
4475
4476 vdb[DBDIR_] = DBDIRCOPY_BAG_DB_ | DBDIRFORCED_
4477 vdb[DBDESC_] += u' Force copied to bag.'
4478 if dd & DBFORCEDIRORIGIN_:
4479
4480 vdb[DBDIR_] = DBDIRCOPY_ORIG_DB_ | DBDIRFORCED_
4481 vdb[DBDESC_] += u' Force copied to origin.'
4482 elif rdc == DBREADNNY_:
4483
4484 vdb[DBDIR_] = DBDIRBAG_DB_NEW_
4485 vdb[DBDESC_] = u'New in origin. Copy to bag, db.'
4486 ron += 1
4487 if _dir & DBFORCEDIRORIGINALL_:
4488 vdb[DBDIR_] = DBDIRORIG_ | DBDIRDELETE_ | DBDIRFORCED_
4489 vdb[DBDESC_] += u' !!! Force ORIGINALL - delete in origin.'
4490 elif _dir & DBFORCEDIRORIGINCOPY_:
4491 vdb[DBDIR_] = DBDIRUNCHANGED_ | DBDIRFORCED_
4492 vdb[DBDESC_] += u' !!! Force ORIGINCOPY - do not modify.'
4493 elif rdc == DBREADYNN_:
4494
4495 vdb[DBDIR_] = DBDIRORIG_DB_NEW_
4496 vdb[DBDESC_] = u'New in bag. Copy to origin.'
4497 rbn += 1
4498 if _dir & DBFORCEDIRBAGALL_:
4499
4500 vdb[DBDIR_] = DBDIRBAG_ | DBDIRDELETE_ | DBDIRFORCED_
4501 vdb[DBDESC_] += u' !!! Force BAGALL - delete in bag.'
4502 elif _dir & DBFORCEDIRBAGCOPY_:
4503
4504 vdb[DBDIR_] = DBDIRUNCHANGED_ | DBDIRFORCED_
4505 vdb[DBDESC_] += u' !!! Force BAGCOPY - do not modify.'
4506 elif rdc == DBREADNYY_:
4507
4508 eq = (vdb[DBDBFLAG_] == vdb[DBORIGFLAG_] and
4509 vdb[DBDBSIZE_] == vdb[DBORIGSIZE_] and
4510 vdb[DBDBLINK_] == vdb[DBORIGLINK_] and
4511 (abs(vdb[DBDBMDATE_] - vdb[DBORIGMDATE_] - TSORIG_O)<=MDDELTA_) )
4512 if eq and vdb[DBCOMPARE_] == P_COMPARECRC32_ and vdb[DBDBFLAG_] != DBFLAGSYMLINK_:
4513
4514 vdb[DBORIGCRC32_] = getcrc32_(vdb[DBORIGPATH_])
4515 eqcrc32 = vdb[DBDBCRC32_] == vdb[DBORIGCRC32_]
4516 eq = eq and eqcrc32
4517 if eq:
4518
4519 vdb[DBDIR_] = DBDIRDELETE_ | DBDIRDB_ | DBDIRORIG_
4520 vdb[DBDESC_] = u'Deleted in bag. Delete in origin.'
4521 rbd += 1
4522 if _dir & DBFORCEDIRBAGALL_ or _dir & DBFORCEDIRBAGCOPY_:
4523
4524 vdb[DBDIR_] = DBDIRBAG_ | DBDIRCOPY_ | DBDIRFORCED_
4525 vdb[DBDESC_] += u' !!! Force BAGCOPY/ALL - copy to bag.'
4526 elif _dir & DBFORCEDIRORIGINCOPY_:
4527
4528 vdb[DBDIR_] = DBDIRUNCHANGED_ | DBDIRFORCED_
4529 vdb[DBDESC_] += u' !!! Force ORIGINCOPY - do not modify.'
4530 else:
4531
4532
4533 vdb[DBDIR_] = DBDIRCONFLICT_
4534 vdb[DBDESC_] = u'Conflict. Deleted in bag, changed in origin.'
4535 rc += 1
4536 dd = _dir
4537 if dd & DBFORCEDIRNEWER_:
4538
4539 if vdb[DBDBMDATE_] - vdb[DBORIGMDATE_] - TSORIG_O < -MDDELTA_:
4540 dd = DBFORCEDIRBAG_
4541 vdb[DBDESC_] += u' Origin is newer.'
4542 elif vdb[DBDBMDATE_] - vdb[DBORIGMDATE_] - TSORIG_O > MDDELTA_:
4543 dd = DBFORCEDIRORIGIN_
4544 vdb[DBDESC_] += u' Bag is newer.'
4545 if dd & DBFORCEDIROLDER_:
4546
4547 if vdb[DBDBMDATE_] - vdb[DBORIGMDATE_] - TSORIG_O < -MDDELTA_:
4548 dd = DBFORCEDIRORIGIN_
4549 vdb[DBDESC_] += u' Bag is older.'
4550 elif vdb[DBDBMDATE_] - vdb[DBORIGMDATE_] - TSORIG_O > MDDELTA_:
4551 dd = DBFORCEDIRBAG_
4552 vdb[DBDESC_] += u' Origin is older.'
4553 if dd & DBFORCEDIRBAGALL_ or dd & DBFORCEDIRBAGCOPY_:
4554 dd = DBFORCEDIRBAG_
4555 vdb[DBDESC_] += u' !!! Force BAGCOPY/ALL.'
4556 if dd & DBFORCEDIRORIGINALL_:
4557 dd = DBFORCEDIRORIGIN_
4558 vdb[DBDESC] += u' !!! Force ORIGINALL.'
4559 if dd & DBFORCEDIRORIGINCOPY_:
4560 dd = DBFORCEDIRNONE_
4561 vdb[DBDIR] = DBDIRUNCHANGED_ | DBDIRFORCED_
4562 vdb[DBDESC_] += u' !!! Force ORIGINCOPY - do not modify.'
4563 if dd & DBFORCEDIRBAG_:
4564
4565 vdb[DBDIR_] = DBDIRCOPY_BAG_DB_ | DBDIRFORCED_
4566 vdb[DBDESC_] += u' Force copied to bag.'
4567 if dd & DBFORCEDIRORIGIN_:
4568
4569 vdb[DBDIR_] = DBDIRDELETE_ | DBDIRDB_ | DBDIRORIG_ | DBDIRFORCED_
4570 vdb[DBDESC_] += u' Force delete in origin.'
4571 elif rdc == DBREADYYN_:
4572
4573 if not back:
4574 eq = (vdb[DBBAGFLAG_] == vdb[DBDBFLAG_] and
4575 vdb[DBBAGSIZE_] == vdb[DBDBSIZE_] and
4576 vdb[DBBAGLINK_] == vdb[DBDBLINK_] and
4577 (abs(vdb[DBDBMDATE_] - vdb[DBBAGMDATE_] - TSBAG_B)<=MDDELTA_) )
4578 if eq and vdb[DBCOMPARE_] == P_COMPARECRC32_ and vdb[DBBAGFLAG_] != DBFLAGSYMLINK_:
4579
4580 vdb[DBBAGCRC32_] = getcrc32_(vdb[DBBAGPATH_])
4581 eq = eq and vdb[DBBAGCRC32_] == vdb[DBDBCRC32_]
4582 else:
4583
4584 eq = True
4585 if eq:
4586
4587 vdb[DBDIR_] = DBDIRDELETE_ | DBDIRBAG_ | DBDIRDB_
4588 vdb[DBDESC_] = u'Deleted in origin. Delete in bag.'
4589 rod += 1
4590 if _dir & DBFORCEDIRORIGINALL_ or _dir & DBFORCEDIRORIGINCOPY_:
4591
4592 vdb[DBDIR_] = DBDIRORIG_ | DBDIRCOPY_ | DBDIRFORCED_
4593 vdb[DBDESC_] += u' !!! Force ORIGINCOPY/ALL - copy ot origin.'
4594 elif _dir & DBFORCEDIRBAGCOPY_:
4595
4596 vdb[DBDIR_] = DBDIRUNCHANGED_ | DBDIRFORCED_
4597 vdb[DBDESC_] += u' !!! Force BAGCOPY - do not modify.'
4598 else:
4599
4600
4601 vdb[DBDIR_] = DBDIRCONFLICT_
4602 vdb[DBDESC_] = u'Conflict. Deleted in origin, changed in bag.'
4603 rc += 1
4604 dd = _dir
4605 if dd & DBFORCEDIRNEWER_:
4606
4607 if vdb[DBDBMDATE_] - vdb[DBBAGMDATE_] - TSBAG_B < -MDDELTA_:
4608 dd = DBFORCEDIRORIGIN_
4609 vdb[DBDESC_] += u' Bag is newer.'
4610 elif vdb[DBDBMDATE_] - vdb[DBBAGMDATE_] - TSBAG_B > MDDELTA_:
4611 dd = DBFORCEDIRBAG_
4612 vdb[DBDESC_] += u' Origin is newer.'
4613 if dd & DBFORCEDIROLDER_:
4614
4615 if vdb[DBDBMDATE_] - vdb[DBBAGMDATE_] - TSBAG_B < -MDDELTA_:
4616 dd = DBFORCEDIRBAG_
4617 vdb[DBDESC_] += u' Origin is older.'
4618 elif vdb[DBDBMDATE_] - vdb[DBBAGMDATE_] - TSBAG_B > MDDELTA_:
4619 dd = DBFORCEDIRORIGIN_
4620 vdb[DBDESC_] += u' Bag is older.'
4621 if dd & DBFORCEDIRORIGINALL or dd & DBFORCEDIRORIGINCOPY:
4622
4623 dd = DBFORCEDIRORIGIN_
4624 vdb[DBDESC_] += u' !!! Force ORIGINCOPY/ALL - copy to origin.'
4625 if dd & DBFORCEDIRBAGCOPY_:
4626
4627 dd = DBFORCEDIRNONE_
4628 vdb[DBDIR_] = DBDIRUNCHANGED_ | DBDIRFORCED_
4629 vdb[DBDESC_] += u' !!! Force BAGCOPY - do not modify.'
4630 if dd & DBFORCEDIRBAG_:
4631
4632 vdb[DBDIR_] = DBDIRDELETE_ | DBDIRDB_ | DBDIRBAG_ | DBDIRFORCED_
4633 vdb[DBDESC_] += u' Force delete in bag.'
4634 if dd & DBFORCEDIRORIGIN_:
4635
4636 vdb[DBDIR_] = DBDIRCOPY_ORIG_DB_ | DBDIRFORCED_
4637 vdb[DBDESC_] += u' Force copy to origin.'
4638 elif rdc == DBREADNYN_:
4639
4640 vdb[DBDIR_] = DBDIRDELETE_ | DBDIRDB_ | DBDIRWARNING_
4641 vdb[DBDESC_] = u'Warning. Item deleted in origin and bag. Will be deleted from DB.'
4642 rbd += 1
4643 rod += 1
4644 rw += 1
4645 elif rdc == DBREADYNY_:
4646
4647 eq = (vdb[DBBAGFLAG_] == vdb[DBORIGFLAG_] and
4648 vdb[DBBAGSIZE_] == vdb[DBORIGSIZE_] and
4649 vdb[DBBAGLINK_] == vdb[DBORIGLINK_] and
4650 (abs(vdb[DBBAGMDATE_] + TSBAG_B - vdb[DBORIGMDATE_] - TSORIG_O) <= MDDELTA_) )
4651 if eq and vdb[DBCOMPARE_] == P_COMPARECRC32_ and vdb[DBBAGFLAG_] != DBFLAGSYMLINK_:
4652
4653 vdb[DBORIGCRC32_] = getcrc32_(vdb[DBORIGPATH_])
4654 vdb[DBBAGCRC32_] = getcrc32_(vdb[DBBAGPATH_])
4655 eq = eq and vdb[DBBAGCRC32_] == vdb[DBORIGCRC32_]
4656 if eq:
4657
4658 vdb[DBDIR_] = DBDIRNEW_ | DBDIRDB_ | DBDIRWARNING_
4659 vdb[DBDESC_] = u'Warning. Exists in bag and origin, new for DB.'
4660 rw += 1
4661 rbn += 1
4662 ron += 1
4663 else:
4664
4665
4666 vdb[DBDIR_] = DBDIRCONFLICT_
4667 vdb[DBDESC_] = u'Conflict. New different items in bag and origin.'
4668 rc += 1
4669 dd = _dir
4670 if dd & DBFORCEDIRNEWER_:
4671
4672 if vdb[DBBAGMDATE_] + TSBAG_B - vdb[DBORIGMDATE_] - TSORIG_O < -MDDELTA_:
4673 dd = DBFORCEDIRBAG_
4674 vdb[DBDESC_] += u' Origin is newer.'
4675 elif vdb[DBBAGMDATE_] + TSBAG_B - vdb[DBORIGMDATE_] - TSORIG_O > MDDELTA_:
4676 dd = DBFORCEDIRORIGIN_
4677 vdb[DBDESC_] += u' Bag is newer.'
4678 if dd & DBFORCEDIROLDER_:
4679
4680 if vdb[DBBAGMDATE_] + TSBAG_B - vdb[DBORIGMDATE_] - TSORIG_O < -MDDELTA_:
4681 dd = DBFORCEDIRORIGIN_
4682 vdb[DBDESC_] += u' Bag is older.'
4683 elif vdb[DBBAGMDATE_] + TSBAG_B - vdb[DBORIGMDATE_] - TSORIG_O > MDDELTA_:
4684 dd = DBFORCEDIRBAG_
4685 vdb[DBDESC_] += u' Origin is older.'
4686 if dd & DBFORCEDIRBAGALL_ or dd & DBFORCEDIRBAGCOPY_:
4687
4688 dd = DBFORCEDIRBAG_
4689 vdb[DBDESC_] += u' !!! Force BAGCOPY/ALL - copy to bag.'
4690 elif dd & DBFORCEDIRORIGINALL_ or dd & DBFORCEDIRORIGINCOPY_:
4691
4692 dd = DBFORCEDIRORIGIN_
4693 vdb[DBDESC_] += u' !!! Force ORIGINCOPY/ALL - copy to origin.'
4694 if dd & DBFORCEDIRBAG_:
4695
4696 vdb[DBDIR_] = DBDIRCOPY_BAG_DB_ | DBDIRFORCED_
4697 vdb[DBDESC_] += u' Force copied to bag.'
4698 if dd & DBFORCEDIRORIGIN_:
4699
4700 vdb[DBDIR_] = DBDIRCOPY_ORIG_DB_ | DBDIRFORCED_
4701 vdb[DBDESC_] += u' Force copied to origin.'
4702 elif rdc == DBREADNNN_:
4703
4704 vdb[DBDIR_] = DBDIRERROR_
4705 vdb[DBDESC_] = u'Error in program - unavailable condition in DB.'
4706 re += 1
4707 else:
4708
4709 vdb[DBDIR_] = DBDIRERROR_
4710 vdb[DBDESC_] = u'Error in program - unavailable condition in case.'
4711 re += 1
4712 if doout:
4713 dbg_(u'f_synccompare: dir=%d, desc="%s".' % (vdb[DBDIR_],vdb[DBDESC_]))
4714
4715 LOCK_[LI_PERCENT_] += LN
4716 log_(u'end f_synccompare')
4717 return (re,rc,rw,rbn,ron,rbc,roc,rbd,rod,rign, rts)
4718
4720 """
4721 Syncronize DB.
4722
4723 Return tuple of modified and errored items.
4724 """
4725 _db_ = cfg.Z_DB
4726 LOCK_ = cfg.LOCK
4727 LI_PERCENT100_ = LI_PERCENT100
4728 LI_DESC_ = LI_DESC
4729 LI_PERCENT_ = LI_PERCENT
4730 LI_CANCEL_ = LI_CANCEL
4731 dbg_ = dbg
4732 path2os_ = path2os
4733 os_path_lexists_ = os.path.lexists
4734 os_utime_ = os.utime
4735 rmdirr_ = rmdirr
4736 shutil_copy2_ = shutil.copy2
4737 getcrc32_ = getcrc32
4738 readlink_ = readlink
4739 os_remove_ = os.remove
4740 symlink_ = symlink
4741 os_makedirs_ = os.makedirs
4742 os_path_dirname_ = os.path.dirname
4743 prt1_ = prt1
4744 prt4_ = prt4
4745 prt5_ = prt5
4746 doout = (cfg.VERBOSE >= 4) or cfg.PYBAG_DEBUG
4747 log_ = log
4748 PYBAG_EMUL_ = cfg.PYBAG_EMUL
4749 DBBAGFLAG_ = DBBAGFLAG
4750 DBBAGSIZE_ = DBBAGSIZE
4751 DBBAGMDATE_ = DBBAGMDATE
4752 DBBAGCRC32_ = DBBAGCRC32
4753 DBBAGREAD_ = DBBAGREAD
4754 DBBAGPATH_ = DBBAGPATH
4755 DBBAGLINK_ = DBBAGLINK
4756 DBDBFLAG_ = DBDBFLAG
4757 DBDBSIZE_ = DBDBSIZE
4758 DBDBMDATE_ = DBDBMDATE
4759 DBDBCRC32_ = DBDBCRC32
4760 DBDBREAD_ = DBDBREAD
4761 DBDBPATH_ = DBDBPATH
4762 DBDBLINK_ = DBDBLINK
4763 DBORIGFLAG_ = DBORIGFLAG
4764 DBORIGSIZE_ = DBORIGSIZE
4765 DBORIGMDATE_ = DBORIGMDATE
4766 DBORIGCRC32_ = DBORIGCRC32
4767 DBORIGREAD_ = DBORIGREAD
4768 DBORIGPATH_ = DBORIGPATH
4769 DBORIGLINK_ = DBORIGLINK
4770 DBDIR_ = DBDIR
4771 DBSYNC_ = DBSYNC
4772 DBCOMPARE_ = DBCOMPARE
4773 DBSYMLINK_ = DBSYMLINK
4774 DBDESC_ = DBDESC
4775 DBDIRMAN_ = DBDIRMAN
4776 DBDESCMAN_ = DBDESCMAN
4777 DBITEMID_ = DBITEMID
4778 DBDIRREADED_ = DBDIRREADED
4779 DBDESCREADED_ = DBDESCREADED
4780 DBREADYES_ = DBREADYES
4781 DBREADNONE_ = DBREADNONE
4782 DBFLAGNONE_ = DBFLAGNONE
4783 DBFLAGDIR_ = DBFLAGDIR
4784 DBFLAGFILE_ = DBFLAGFILE
4785 DBFLAGSYMLINK_ = DBFLAGSYMLINK
4786 DBFLAGUNKNOWN_ = DBFLAGUNKNOWN
4787 DBMDATENONE_ = DBMDATENONE
4788 DBNONE_ = DBNONE
4789 DBPATHNONE_ = DBPATHNONE
4790 DBDESCNONE_ = DBDESCNONE
4791 DBDESCMANNONE_ = DBDESCMANNONE
4792 DBDESCOK_ = DBDESCOK
4793 DBLINKNONE_ = DBLINKNONE
4794 DBITEMNONE_ = DBITEMNONE
4795 DBDIRNONE_ = DBDIRNONE
4796 DBDIRBAG_ = DBDIRBAG
4797 DBDIRORIG_ = DBDIRORIG
4798 DBDIRDB_ = DBDIRDB
4799 DBDIRCOPY_ = DBDIRCOPY
4800 DBDIRDELETE_ = DBDIRDELETE
4801 DBDIRWARNING_ = DBDIRWARNING
4802 DBDIRCONFLICT_ = DBDIRCONFLICT
4803 DBDIRERROR_ = DBDIRERROR
4804 DBDIRNEW_ = DBDIRNEW
4805 DBDIRUNCHANGED_ = DBDIRUNCHANGED
4806 DBDIRFORCED_ = DBDIRFORCED
4807 DBDIRIGNORED_ = DBDIRIGNORED
4808 DBDIRDIFF_ = DBDIRDIFF
4809 DBDIROK_ = DBDIROK
4810 DBDIRRESTORE_ = DBDIRRESTORE
4811 DBSYNCNONE_ = DBSYNCNONE
4812 DBSYNCERRREADBAG_ = DBSYNCERRREADBAG
4813 DBSYNCERRREADSYNC_ = DBSYNCERRREADSYNC
4814 DBSYNCERRREAD_ = DBSYNCERRREAD
4815 DBSYNCERRWRITEBAG_ = DBSYNCERRWRITEBAG
4816 DBSYNCERRWRITESYNC_ = DBSYNCERRWRITESYNC
4817 DBSYNCERRWRITE_ = DBSYNCERRWRITE
4818 DBSYNCOK_ = DBSYNCOK
4819 DBSYNCERROTHER_ = DBSYNCERROTHER
4820 DBSYNCERRCOMPARE_ = DBSYNCERRCOMPARE
4821 DBSYNCERROR_ = DBSYNCERROR
4822 DBSYNCCOMPARE_ = DBSYNCCOMPARE
4823 DBSYNCIGNORED_ = DBSYNCIGNORED
4824 DBSYNCCONFLICT_ = DBSYNCCONFLICT
4825 DBFORCEDIRNONE_ = DBFORCEDIRNONE
4826 DBFORCEDIRBAG_ = DBFORCEDIRBAG
4827 DBFORCEDIRORIGIN_ = DBFORCEDIRORIGIN
4828 DBFORCEDIROLDER_ = DBFORCEDIROLDER
4829 DBFORCEDIRNEWER_ = DBFORCEDIRNEWER
4830 DBFORCEDIRBAGALL_ = DBFORCEDIRBAGALL
4831 DBFORCEDIRORIGINALL_ = DBFORCEDIRORIGINALL
4832 DBFORCEDIRORIGINCOPY_ = DBFORCEDIRORIGINCOPY
4833 DBFORCEDIRBAGCOPY_ = DBFORCEDIRBAGCOPY
4834 P_COMPARECRC32_ = P_COMPARECRC32
4835 DBSYMLINKEMUL_ = DBSYMLINKEMUL
4836 _u_ = z_u
4837
4838 TSDIR_ = cfg.TSDIR
4839 TSDIRBAG_ = TSDIRBAG
4840 TSDIRORIG_ = TSDIRORIG
4841 TSDIRNONE_ = TSDIRNONE
4842 TSBAG_ = cfg.TSBAG
4843 TSORIG_ = cfg.TSORIG
4844 DBDIRTS_ = DBDIRTS
4845 tsdirnone = TSDIR_ == TSDIRNONE_
4846 if TSDIR_ == TSDIRBAG_:
4847 tsdesc = u'Unchanged. Time shift to bag.'
4848 else:
4849 tsdesc = u'Unchanged. Time shift to origin.'
4850
4851 log_(u'start f_syncsync')
4852 prt4_(u'\nSYNCHRONIZE\n')
4853 sn = 0
4854 sc = 0
4855 sd = 0
4856 ts = 0
4857 se = 0
4858 su = 0
4859 if PYBAG_EMUL_:
4860 prt5_(u'f_syncsync: Emulation on.')
4861 prt5_(u'f_syncsync: START synchronizing.')
4862 ks = cfg.Z_KS
4863 LOCK_[LI_PERCENT100_] = len(_db_) * 2 + 1
4864 LOCK_[LI_DESC_] = u'Synchronize files.'
4865 LOCK_[LI_PERCENT_] = 0
4866 if LOCK_[LI_CANCEL_]:
4867 log_(u'f_syncsync: canceled.')
4868 return (sd, sn, sc, su, se)
4869
4870
4871 prt4_(u' Delete processing.\n')
4872 dirflg = DBDIRCONFLICT_ | DBDIRERROR_ | DBDIRIGNORED_ | DBDIRDELETE_
4873 for k in reversed(ks):
4874 if doout:
4875 prt4_(u' process d(%s).' % (path2os_(k),), place=u'f_syncsync')
4876 LOCK_[LI_PERCENT_] += 1
4877 if LOCK_[LI_CANCEL_]:
4878 log_(u'f_syncsync: canceled.')
4879 return (sd, sn, sc, su, se)
4880 vdb = _db_[k]
4881 vdb[DBDESC_] = u''
4882 vdb_DBDIR_ = vdb[DBDIR_]
4883 if not (vdb_DBDIR_ & dirflg):
4884 continue
4885 elif vdb_DBDIR_ & DBDIRCONFLICT_:
4886
4887
4888 vdb[DBSYNC_] |= DBSYNCCONFLICT_
4889 if doout:
4890 prt4_(u'\t\t previous conflict when comparing was.', place=u'f_syncsync')
4891 su += 1
4892 elif vdb_DBDIR_ & DBDIRERROR_:
4893
4894
4895 vdb[DBSYNC_] |= DBSYNCERRCOMPARE_
4896 if doout:
4897 prt4_(u'\t\t previous error when comparing was.', place=u'f_syncsync')
4898 su += 1
4899 elif vdb[DBSYNC_] & DBSYNCERROR_:
4900
4901 if doout:
4902 prt4_(u'\t\t previous error in sync.', place=u'f_syncsync')
4903 su += 1
4904 elif not (vdb[DBSYNC_] & DBSYNCCOMPARE_):
4905
4906 continue
4907 elif (vdb_DBDIR_ & DBDIRIGNORED_):
4908
4909 vdb[DBSYNC_] |= DBSYNCIGNORED_
4910 vdb[DBDESC_] += u' OK. Ignored.'
4911 elif vdb_DBDIR_ & DBDIRDELETE_:
4912
4913 if doout:
4914 prt4_(u'\t\t DELETE.', place=u'f_syncsync')
4915 if vdb_DBDIR_ & DBDIRORIG_:
4916
4917 vdb_DBORIGPATH_ = vdb[DBORIGPATH_]
4918 vdb_DBORIGFLAG_ = vdb[DBORIGFLAG_]
4919 if (vdb_DBORIGFLAG_ == DBFLAGFILE_) or (vdb_DBORIGFLAG_ == DBFLAGSYMLINK_):
4920
4921 try:
4922 if doout:
4923 prt4_(u'\t\t*** remove ORIG(%s).' % (vdb_DBORIGPATH_,), place=u'f_syncsync')
4924 if not PYBAG_EMUL_:
4925 os.remove(vdb_DBORIGPATH_)
4926 vdb[DBSYNC_] = DBSYNCOK_
4927 vdb[DBDESC_] += u' OK. Deleted in origin.'
4928 sd += 1
4929 except Exception, exc:
4930 se += 1
4931 vdb[DBSYNC_] |= DBSYNCERRWRITESYNC_
4932 vdb[DBDESC_] += u' Can\'t remove origin file or symlink.'
4933 prt1_(u'<R>ERROR: Can\'t remove origin file or symlink "%s". "%s".</R>' %
4934 (vdb_DBORIGPATH_, _u_(exc)), place=u'f_syncsync', error=exc)
4935 elif vdb_DBORIGFLAG_ == DBFLAGDIR_:
4936
4937 try:
4938 if doout:
4939 prt4_(u'\t\t*** rmdir ORIG(%s).' % (vdb_DBORIGPATH_,), place=u'f_syncsync')
4940 if not PYBAG_EMUL_:
4941 rmdirr(vdb_DBORIGPATH_)
4942 vdb[DBSYNC_] = DBSYNCOK_
4943 vdb[DBDESC_] += u' OK. Deleted in origin.'
4944 sd += 1
4945 except Exception, exc:
4946 se += 1
4947 vdb[DBSYNC_] |= DBSYNCERRWRITESYNC_
4948 vdb[DBDESC_] += u' Can\'t remove origin directory.'
4949 prt1_(u'<R>ERROR: Can\'t remove origin directory "%s". "%s".</R>' %
4950 (vdb_DBORIGPATH_, _u_(exc)), place=u'f_syncsync', error=exc)
4951 else:
4952
4953 se += 1
4954 vdb[DBSYNC_] |= DBSYNCERROTHER_
4955 vdb[DBDESC_] += u'Error in program. Unavailable condition then remove origin.'
4956 prt1_(u'<R>ERROR: Unavailable condition then remove origin "%s", dirflag(%s).</R>' %
4957 (vdb[DBORIGPATH_], vdb[DBORIGFLAG_]), place=u'f_syncsync')
4958 if (vdb_DBDIR_ & DBDIRBAG_) and not (vdb[DBSYNC_] & DBSYNCERROR_):
4959
4960 vdb_DBBAGPATH_ = vdb[DBBAGPATH_]
4961 vdb_DBBAGFLAG_ = vdb[DBBAGFLAG_]
4962 if (vdb_DBBAGFLAG_ == DBFLAGFILE_) or (vdb_DBBAGFLAG_ == DBFLAGSYMLINK_):
4963
4964 try:
4965 if doout:
4966 prt4_(u'\t\t*** remove BAG(%s).' % (vdb_DBBAGPATH_,), place=u'f_syncsync')
4967 if not PYBAG_EMUL_:
4968 os.remove(vdb_DBBAGPATH_)
4969 vdb[DBSYNC_] = DBSYNCOK_
4970 vdb[DBDESC_] += u' OK. Deleted in bag.'
4971 sd += 1
4972 except Exception, exc:
4973 se += 1
4974 vdb[DBSYNC_] |= DBSYNCERRWRITEBAG_
4975 vdb[DBDESC_] += u' Can\'t remove bag file or symlink.'
4976 prt1_(u'<R>ERROR: Can\'t remove bag file or symlink "%s". "%s".</R>' %
4977 (vdb_DBBAGPATH_, _u_(exc)), place=u'f_syncsync', error=exc)
4978 elif vdb_DBBAGFLAG_ == DBFLAGDIR_:
4979
4980 try:
4981 if doout:
4982 prt4_(u'\t\t*** rmdir BAG(%s).' % (vdb_DBBAGPATH_,), place=u'f_syncsync')
4983 if not PYBAG_EMUL_:
4984 rmdirr(vdb_DBBAGPATH_)
4985 vdb[DBSYNC_] = DBSYNCOK_
4986 vdb[DBDESC_] += u' OK. Deleted in bag.'
4987 sd += 1
4988 except Exception, exc:
4989 se += 1
4990 vdb[DBSYNC_] |= DBSYNCERRWRITEBAG_
4991 vdb[DBDESC_] += u' Can\'t remove bag directory.'
4992 prt1_(u'<R>ERROR: Can\'t remove bag directory "%s". "%s".</R>' %
4993 (vdb_DBBAGPATH_, _u_(exc)), place=u'f_syncsync', error=exc)
4994 else:
4995
4996 se += 1
4997 vdb[DBSYNC_] |= DBSYNCERROTHER_
4998 vdb[DBDESC_] += u'Error in program. Unavailable condition then remove bag.'
4999 prt1_(u'<R>f_syncsync: ERROR: Unavailable condition then remove bag "%s", flag(%s).</R>' %
5000 (vdb_DBBAGPATH_, vdb[DBBAGFLAG_]), place=u'f_syncsync')
5001 if (vdb_DBDIR_ & DBDIRDB_) and not (vdb[DBSYNC_] & DBSYNCERROR_):
5002
5003 vdb[DBSYNC_] = DBSYNCOK_
5004 vdb[DBDESC_] += u' OK. Deleted in DB.'
5005
5006 if doout:
5007 prt4_(u'\t\t*** remove DB(%s).' % (vdb[DBDBPATH_],), place=u'f_syncsync')
5008
5009
5010
5011 prt4_(u'\n Copy, new processing.\n')
5012
5013 for k in ks:
5014 if doout:
5015 prt4_(u' process cn(%s).' % (path2os_(k),), place=u'f_syncsync')
5016 LOCK_[LI_PERCENT_] += 1
5017 if LOCK_[LI_CANCEL_]:
5018 log_(u'f_syncsync: canceled.')
5019 return (sd, sn, sc, su, se)
5020 vdb = _db_[k]
5021
5022
5023 vdb_DBDIR_ = vdb[DBDIR_]
5024 if vdb_DBDIR_ & DBDIRUNCHANGED_:
5025
5026 if tsdirnone or not (vdb_DBDIR_ & DBDIRTS_):
5027 vdb[DBSYNC_] = DBSYNCOK_
5028 vdb[DBDESC_] += u' OK. Unchanged.'
5029 else:
5030
5031 if TSDIR_ == TSDIRBAG_:
5032
5033 try:
5034 t = vdb[DBORIGMDATE_]
5035 if not PYBAG_EMUL_ and vdb[DBBAGFLAG_] == DBFLAGFILE_:
5036 os_utime_(vdb[DBBAGPATH_],(t,t))
5037 vdb[DBBAGMDATE_] = vdb[DBDBMDATE_] = t
5038 vdb[DBDESC_] += tsdesc
5039 vdb[DBSYNC_] = DBSYNCOK_
5040 ts += 1
5041 if doout:
5042 prt4_(u'\t\t*** timeshift BAG(%s).' % (vdb[DBBAGPATH_],), place=u'f_syncsync')
5043 except Exception, exc:
5044 se += 1
5045 vdb[DBSYNC_] |= DBSYNCERRWRITESYNC_
5046 vdb[DBDESC_] += u' Can\'t time shift bag file.'
5047 prt1(u'<R>ERROR: Can\'t time shift bag file "%s". "%s"</R>' %
5048 (vdb[DBBAGPATH_], _u_(exc)), place=u'f_syncsync',error=exc)
5049 else:
5050
5051 try:
5052 t = vdb[DBBAGMDATE_]
5053 if not PYBAG_EMUL_ and vdb[DBORIGFLAG_] == DBFLAGFILE_:
5054 os_utime_(vdb[DBORIGPATH_],(t,t))
5055 vdb[DBORIGMDATE_] = vdb[DBDBMDATE_] = t
5056 vdb[DBDESC_] += tsdesc
5057 vdb[DBSYNC_] = DBSYNCOK_
5058 ts += 1
5059 if doout:
5060 prt4_(u'\t\t*** timeshift ORIG(%s).' % (vdb[DBORIGPATH_],), place=u'f_syncsync')
5061 except Exception, exc:
5062 se += 1
5063 vdb[DBSYNC_] |= DBSYNCERRWRITESYNC_
5064 vdb[DBDESC_] += u' Can\'t time shift origin file.'
5065 prt1(u'<R>ERROR: Can\'t time shift origin file "%s". "%s"</R>' %
5066 (vdb[DBORIGPATH_], _u_(exc)), place=u'f_syncsync',error=exc)
5067 continue
5068 cfg.SYMLINKEMUL = vdb[DBSYMLINKEMUL_]
5069 if vdb_DBDIR_ & dirflg or not (vdb[DBSYNC_] & DBSYNCCOMPARE_) or (vdb[DBSYNC_] & DBSYNCERROR_):
5070 continue
5071 elif vdb_DBDIR_ & DBDIRCOPY_:
5072
5073 if doout:
5074 prt4_(u'\t\t COPY.', place=u'f_syncsync')
5075 vdb_DBORIGPATH_ = vdb[DBORIGPATH_]
5076 vdb_DBBAGPATH_ = vdb[DBBAGPATH_]
5077 if vdb_DBDIR_ & DBDIRORIG_:
5078
5079 vdb_DBBAGFLAG_ = vdb[DBBAGFLAG_]
5080 if (vdb_DBBAGFLAG_ == DBFLAGFILE_) or (vdb_DBBAGFLAG_ == DBFLAGSYMLINK_):
5081
5082 try:
5083 if doout:
5084 prt4_(u'\t\t*** copyfile/sym ORIG(%s).' % (vdb_DBORIGPATH_,), place=u'f_syncsync')
5085
5086 if vdb[DBORIGFLAG_] == DBFLAGDIR_:
5087
5088 if not PYBAG_EMUL_:
5089 if os_path_lexists_(vdb_DBORIGPATH_):
5090 rmdirr_(vdb_DBORIGPATH_)
5091 if not PYBAG_EMUL_:
5092 if vdb_DBBAGFLAG_ == DBFLAGFILE_:
5093
5094 shutil_copy2_(vdb_DBBAGPATH_, vdb_DBORIGPATH_)
5095 if vdb[DBCOMPARE_] == P_COMPARECRC32_:
5096 vdb[DBBAGCRC32_] = getcrc32_(vdb_DBBAGPATH_)
5097 else:
5098
5099 lp = readlink_(vdb_DBBAGPATH_)
5100
5101 if os_path_lexists_(vdb_DBORIGPATH_):
5102 os_remove_(vdb_DBORIGPATH_)
5103 symlink_(lp, vdb_DBORIGPATH_)
5104 vdb[DBBAGCRC32_] = DBNONE_
5105 vdb[DBORIGFLAG_] = vdb_DBBAGFLAG_
5106 vdb[DBORIGSIZE_] = vdb[DBBAGSIZE_]
5107 vdb[DBORIGMDATE_] = vdb[DBBAGMDATE_]
5108 vdb[DBORIGCRC32_] = vdb[DBBAGCRC32_]
5109 vdb[DBORIGLINK_] = vdb[DBBAGLINK_]
5110 vdb[DBSYNC_] = DBSYNCOK_
5111 vdb[DBDESC_] += u' OK. Copied to origin.'
5112 sc += 1
5113 except Exception, exc:
5114 se += 1
5115 vdb[DBSYNC_] |= DBSYNCERRWRITESYNC_
5116 vdb[DBDESC_] += u' Can\'t remove/copy origin file/sym.'
5117 prt1(u'<R>ERROR: Can\'t remove/copy origin file/sym "%s". "%s"</R>' %
5118 (vdb_DBORIGPATH_, _u_(exc)), place=u'f_syncsync',error=exc)
5119
5120
5121 elif vdb_DBBAGFLAG_ == DBFLAGDIR_:
5122
5123 if doout:
5124 prt4_(u'\t\t*** copydir ORIG(%s).' % (vdb_DBORIGPATH_,), place=u'f_syncsync')
5125 try:
5126
5127 if (vdb[DBORIGFLAG_] == DBFLAGFILE_) or (vdb[DBORIGFLAG_] == DBFLAGSYMLINK_):
5128
5129 if not PYBAG_EMUL_:
5130 if os_path_lexists_(vdb_DBORIGPATH_):
5131 os_remove_(vdb_DBORIGPATH_)
5132 if not PYBAG_EMUL_ and not os_path_lexists_(vdb_DBORIGPATH_):
5133 os_makedirs_(vdb_DBORIGPATH_)
5134 vdb[DBBAGCRC32_] = DBNONE_
5135 vdb[DBORIGFLAG_] = vdb_DBBAGFLAG_
5136 vdb[DBORIGSIZE_] = vdb[DBBAGSIZE_]
5137 vdb[DBORIGMDATE_] = vdb[DBBAGMDATE_]
5138 vdb[DBORIGCRC32_] = vdb[DBBAGCRC32_]
5139 vdb[DBORIGLINK_] = vdb[DBBAGLINK_]
5140 vdb[DBSYNC_] = DBSYNCOK_
5141 vdb[DBDESC_] += u' OK. Copied to origin.'
5142 sc += 1
5143 except Exception, exc:
5144 se += 1
5145 vdb[DBSYNC_] |= DBSYNCERRWRITESYNC_
5146 vdb[DBDESC_] += u' Can\'t remove/make origin file or symlink for copy directory.'
5147 prt1_(u'<R>ERROR: Can\'t remove/make origin file or symlink for copy directory "%s". %s.</R>' %
5148 (vdb_DBORIGPATH_, _u_(exc)), place=u'f_syncsync',error=exc)
5149
5150 else:
5151
5152 se += 1
5153 vdb[DBSYNC_] |= DBSYNCERROTHER_
5154 vdb[DBDESC_] += u'Error in program. Unavailable condition then copy origin.'
5155 prt1(u'<R>ERROR: Unavailable condition then copy origin "%s", flag(%s).</R>' %
5156 (vdb_DBORIGPATH_, vdb[DBORIGFLAG_]), place=u'f_syncsync')
5157 continue
5158 elif vdb_DBDIR_ & DBDIRBAG_:
5159
5160 vdb_DBORIGFLAG_ = vdb[DBORIGFLAG_]
5161 if (vdb_DBORIGFLAG_ == DBFLAGFILE_) or (vdb_DBORIGFLAG_ == DBFLAGSYMLINK_):
5162
5163 try:
5164 if doout:
5165 prt4_(u'\t\t*** copyfile/sym BAG(%s).' % (vdb_DBBAGPATH_,), place=u'f_syncsync')
5166
5167 if vdb[DBBAGFLAG_] == DBFLAGDIR_:
5168
5169 if not PYBAG_EMUL_:
5170 if os_path_lexists_(vdb_DBBAGPATH_):
5171 rmdirr_(vdb_DBBAGPATH_)
5172 if not PYBAG_EMUL_:
5173 if vdb_DBORIGFLAG_ == DBFLAGFILE_:
5174
5175 shutil_copy2_(vdb_DBORIGPATH_, vdb_DBBAGPATH_)
5176 if vdb[DBCOMPARE_] == P_COMPARECRC32_:
5177 vdb[DBORIGCRC32_] = getcrc32_(vdb_DBORIGPATH_)
5178 else:
5179
5180 lp = readlink_(vdb_DBORIGPATH_)
5181
5182 if os_path_lexists_(vdb_DBBAGPATH_):
5183 os_remove_(vdb_DBBAGPATH_)
5184 symlink_(lp, vdb_DBBAGPATH_)
5185 vdb[DBORIGCRC32_] = DBNONE_
5186 vdb[DBBAGFLAG_] = vdb_DBORIGFLAG_
5187 vdb[DBBAGSIZE_] = vdb[DBORIGSIZE_]
5188 vdb[DBBAGMDATE_] = vdb[DBORIGMDATE_]
5189 vdb[DBBAGCRC32_] = vdb[DBORIGCRC32_]
5190 vdb[DBBAGLINK_] = vdb[DBORIGLINK_]
5191 vdb[DBSYNC_] = DBSYNCOK_
5192 vdb[DBDESC_] += u' OK. Copied to bag.'
5193 sc += 1
5194 except Exception, exc:
5195 se += 1
5196 vdb[DBSYNC_] |= DBSYNCERRWRITEBAG_
5197 vdb[DBDESC_] += u' Can\'t remove/copy bag file/sym.'
5198 prt1_(u'<R>ERROR: Can\'t remove/copy bag file/sym "%s". "%s"</R>' %
5199 (vdb_DBBAGPATH_, _u_(exc)), place=u'f_syncsync',error=exc)
5200
5201
5202 elif vdb_DBORIGFLAG_ == DBFLAGDIR_:
5203
5204 if doout:
5205 prt4_(u'\t\t*** copydir BAG(%s).' % (vdb_DBBAGPATH_,), place=u'f_syncsync')
5206 try:
5207
5208 if (vdb[DBBAGFLAG_] == DBFLAGFILE_) or (vdb[DBBAGFLAG_] == DBFLAGSYMLINK_):
5209
5210 if not PYBAG_EMUL_:
5211 if os_path_lexists_(vdb_DBBAGPATH_):
5212 os_remove_(vdb_DBBAGPATH_)
5213 if not PYBAG_EMUL_ and not os_path_lexists_(vdb_DBBAGPATH_):
5214 os_makedirs_(vdb_DBBAGPATH_)
5215 vdb[DBORIGCRC32_] = DBNONE_
5216 vdb[DBBAGFLAG_] = vdb_DBORIGFLAG_
5217 vdb[DBBAGSIZE_] = vdb[DBORIGSIZE_]
5218 vdb[DBBAGMDATE_] = vdb[DBORIGMDATE_]
5219 vdb[DBBAGCRC32_] = vdb[DBORIGCRC32_]
5220 vdb[DBBAGLINK_] = vdb[DBORIGLINK_]
5221 vdb[DBSYNC_] = DBSYNCOK_
5222 vdb[DBDESC_] += u' OK. Copied to bag.'
5223 sc += 1
5224 except Exception, exc:
5225 se += 1
5226 vdb[DBSYNC_] |= DBSYNCERRWRITEBAG_
5227 vdb[DBDESC_] += u' Can\'t remove/make bag file or symlink for copy directory.'
5228 prt1_(u'<R>ERROR: Can\'t remove/make bag file or symlink for copy directory "%s". %s</R>' %
5229 (vdb_DBBAGPATH_, _u_(exc)), place=u'f_syncsync',error=exc)
5230 else:
5231
5232 se += 1
5233 vdb[DBSYNC_] |= DBSYNCERROTHER_
5234 vdb[DBDESC_] += u'Error in program. Unavailable condition then copy bag.'
5235 prt1_(u'<R>ERROR: Unavailable condition then copy bag "%s", flag(%s).</R>' %
5236 (vdb_DBORIGPATH_, vdb_DBORIGFLAG_), place=u'f_syncsync')
5237 continue
5238 if (vdb_DBDIR_ & DBDIRDB_) and not (vdb[DBSYNC_] & DBSYNCERROR_):
5239
5240 vdb[DBDBFLAG_] = vdb[DBORIGFLAG_]
5241 vdb[DBDBSIZE_] = vdb[DBORIGSIZE_]
5242 vdb[DBDBMDATE_] = vdb[DBORIGMDATE_]
5243 vdb[DBDBCRC32_] = vdb[DBORIGCRC32_]
5244 vdb[DBDBLINK_] = vdb[DBORIGLINK_]
5245 vdb[DBSYNC_] = DBSYNCOK_
5246 vdb[DBDESC_] += u' OK. Copied to DB.'
5247
5248 if doout:
5249 prt4_(u'\t\t*** copyfile/sym DB(%s).' % (k,), place=u'f_syncsync')
5250 elif vdb_DBDIR_ & DBDIRNEW_:
5251
5252 vdb_DBORIGPATH_ = vdb[DBORIGPATH_]
5253 vdb_DBBAGPATH_ = vdb[DBBAGPATH_]
5254 if doout:
5255 prt4_(u'\t\t NEW.', place=u'f_syncsync')
5256 if vdb_DBDIR_ & DBDIRORIG_:
5257
5258 vdb_DBBAGFLAG_ = vdb[DBBAGFLAG_]
5259 if (vdb_DBBAGFLAG_ == DBFLAGFILE_) or (vdb_DBBAGFLAG_ == DBFLAGSYMLINK_):
5260
5261 try:
5262 if doout:
5263 prt4_(u'\t\t*** makefile/sym ORIG(%s).' % (vdb_DBORIGPATH_,), place=u'f_syncsync')
5264 if not PYBAG_EMUL_:
5265
5266 pdn = os_path_dirname_(vdb_DBORIGPATH_)
5267 if not os_path_lexists_(pdn):
5268
5269 os_makedirs_(pdn)
5270 if vdb_DBBAGFLAG_ == DBFLAGFILE_:
5271
5272 shutil_copy2_(vdb_DBBAGPATH_, vdb_DBORIGPATH_)
5273 if vdb[DBCOMPARE_] == P_COMPARECRC32_:
5274 vdb[DBBAGCRC32_] = getcrc32_(vdb_DBBAGPATH_)
5275 else:
5276
5277 lp = readlink_(vdb_DBBAGPATH_)
5278
5279 if os_path_lexists_(vdb_DBORIGPATH_):
5280 os_remove_(vdb_DBORIGPATH_)
5281 symlink_(lp, vdb_DBORIGPATH_)
5282 vdb[DBBAGCRC32_] = DBNONE_
5283 vdb[DBORIGFLAG_] = vdb_DBBAGFLAG_
5284 vdb[DBORIGSIZE_] = vdb[DBBAGSIZE_]
5285 vdb[DBORIGMDATE_] = vdb[DBBAGMDATE_]
5286 vdb[DBORIGCRC32_] = vdb[DBBAGCRC32_]
5287 vdb[DBORIGLINK_] = vdb[DBBAGLINK_]
5288 vdb[DBSYNC_] = DBSYNCOK_
5289 vdb[DBDESC_] += u' OK. Created in origin.'
5290 sn += 1
5291 except Exception, exc:
5292 se += 1
5293 vdb[DBSYNC_] |= DBSYNCERRWRITESYNC_
5294 vdb[DBDESC_] += u' Can\'t create origin file/sym.'
5295 prt1_(u'<R>ERROR: Can\'t create origin file/sym "%s". %s</R>' %
5296 (vdb_DBORIGPATH_, _u_(exc)), place=u'f_syncsync',error=exc)
5297
5298
5299 elif vdb_DBBAGFLAG_ == DBFLAGDIR_:
5300
5301 try:
5302 if doout:
5303 prt4_(u'\t\t*** makedirs ORIG(%s).' % (vdb_DBORIGPATH_,), place=u'f_syncsync')
5304 if not PYBAG_EMUL_:
5305 if not os_path_lexists_(vdb_DBORIGPATH_):
5306
5307 os_makedirs_(vdb_DBORIGPATH_)
5308 vdb[DBBAGCRC32_] = DBNONE_
5309 vdb[DBORIGFLAG_] = vdb_DBBAGFLAG_
5310 vdb[DBORIGSIZE_] = vdb[DBBAGSIZE_]
5311 vdb[DBORIGMDATE_] = vdb[DBBAGMDATE_]
5312 vdb[DBORIGCRC32_] = vdb[DBBAGCRC32_]
5313 vdb[DBORIGLINK_] = vdb[DBBAGLINK_]
5314 vdb[DBSYNC_] = DBSYNCOK_
5315 vdb[DBDESC_] += u' OK. Created in origin.'
5316 sn += 1
5317 except Exception, exc:
5318 se += 1
5319 vdb[DBSYNC_] |= DBSYNCERRWRITESYNC_
5320 vdb[DBDESC_] += u' Can\'t create origin directory.'
5321 prt1_(u'<R>ERROR: Can\'t create origin directory "%s". "%s"</R>' %
5322 (vdb_DBORIGPATH_, _u_(exc)), place=u'f_syncsync', error=exc)
5323 else:
5324
5325 se += 1
5326 vdb[DBSYNC_] |= DBSYNCERROTHER_
5327 vdb[DBDESC_] += u'Error in program. Unavailable condition then create origin.'
5328 prt1_(u'<R>f_syncsync: ERROR: Unavailable condition then create origin "%s", flag(%s).</R>' %
5329 (vdb_DBORIGPATH_,vdb[DBORIGFLAG_]))
5330 continue
5331 elif vdb_DBDIR_ & DBDIRBAG_:
5332
5333 vdb_DBORIGFLAG_ = vdb[DBORIGFLAG_]
5334 if (vdb_DBORIGFLAG_ == DBFLAGFILE_) or (vdb_DBORIGFLAG_ == DBFLAGSYMLINK_):
5335
5336 try:
5337 if doout:
5338 prt4_(u'\t\t*** makefile/sym BAG(%s).' % (vdb_DBBAGPATH_,), place=u'f_syncsync')
5339 if not PYBAG_EMUL_:
5340
5341 pdn = os_path_dirname_(vdb_DBBAGPATH_)
5342 if not os_path_lexists_(pdn):
5343
5344 os_makedirs_(pdn)
5345 if vdb_DBORIGFLAG_ == DBFLAGFILE_:
5346
5347 shutil_copy2_(vdb_DBORIGPATH_, vdb_DBBAGPATH_)
5348 if vdb[DBCOMPARE_] == P_COMPARECRC32_:
5349 vdb[DBORIGCRC32_] = getcrc32_(vdb_DBORIGPATH_)
5350 else:
5351
5352 lp = readlink_(vdb_DBORIGPATH_)
5353
5354 if os_path_lexists_(vdb_DBBAGPATH_):
5355 os_remove_(vdb_DBBAGPATH_)
5356 symlink_(lp, vdb_DBBAGPATH_)
5357 vdb[DBORIGCRC32_] = DBNONE_
5358 vdb[DBBAGFLAG_] = vdb_DBORIGFLAG_
5359 vdb[DBBAGSIZE_] = vdb[DBORIGSIZE_]
5360 vdb[DBBAGMDATE_] = vdb[DBORIGMDATE_]
5361 vdb[DBBAGCRC32_] = vdb[DBORIGCRC32_]
5362 vdb[DBBAGLINK_] = vdb[DBORIGLINK_]
5363 vdb[DBSYNC_] = DBSYNCOK_
5364 vdb[DBDESC_] += u' OK. Created in bag.'
5365 sn += 1
5366 except Exception, exc:
5367 se += 1
5368 vdb[DBSYNC_] |= DBSYNCERRWRITEBAG_
5369 vdb[DBDESC_] += u' Can\'t create bag file/sym.'
5370 prt1(u'<R>f_syncsync: ERROR: Can\'t create bag file/sym "%s". "%s"</R>' %
5371 (vdb_DBBAGPATH_, _u_(exc)), place=u'f_syncsync', error=exc)
5372
5373
5374 elif vdb_DBORIGFLAG_ == DBFLAGDIR_:
5375
5376 try:
5377 if doout:
5378 prt4_(u'\t\t*** makedirs BAG(%s).' % (vdb_DBBAGPATH_,), place=u'f_syncsync')
5379 if not PYBAG_EMUL_:
5380 if not os_path_lexists_(vdb_DBBAGPATH_):
5381
5382 os_makedirs_(vdb_DBBAGPATH_)
5383 vdb[DBORIGCRC32_] = DBNONE_
5384 vdb[DBBAGFLAG_] = vdb_DBORIGFLAG_
5385 vdb[DBBAGSIZE_] = vdb[DBORIGSIZE_]
5386 vdb[DBBAGMDATE_] = vdb[DBORIGMDATE_]
5387 vdb[DBBAGCRC32_] = vdb[DBORIGCRC32_]
5388 vdb[DBBAGLINK_] = vdb[DBORIGLINK_]
5389 vdb[DBSYNC_] = DBSYNCOK_
5390 vdb[DBDESC_] += u' OK. Created in bag.'
5391 sn += 1
5392 except Exception, exc:
5393 se += 1
5394 vdb[DBSYNC_] |= DBSYNCERRWRITEBAG_
5395 vdb[DBDESC_] += u' Can\'t create bag directory.'
5396 prt1(u'<R>f_syncsync: ERROR: Can\'t create bag directory "%s". "%s"</R>' %
5397 (vdb_DBBAGPATH_, _u_(exc)), place=u'f_syncsync',error=exc)
5398 else:
5399
5400 se += 1
5401 vdb[DBSYNC_] |= DBSYNCERROTHER_
5402 vdb[DBDESC_] += u'Error in program. Unavailable condition then create origin.'
5403 prt1(u'<R>f_syncsync: ERROR: Unavailable condition then create origin "%s", flag(%s).</R>' %
5404 (vdb_DBORIGPATH_, vdb[DBORIGFLAG_]), place=u'f_syncsync')
5405 continue
5406 if (vdb_DBDIR_ & DBDIRDB_) and not (vdb[DBSYNC_] & DBSYNCERROR_):
5407
5408 if doout:
5409 prt4_(u'\t\t Create item in DB.', place=u'f_syncsync')
5410 vdb[DBDBFLAG_] = vdb[DBORIGFLAG_]
5411 vdb[DBDBSIZE_] = vdb[DBORIGSIZE_]
5412 vdb[DBDBMDATE_] = vdb[DBORIGMDATE_]
5413 vdb[DBDBCRC32_] = vdb[DBORIGCRC32_]
5414 vdb[DBDBLINK_] = vdb[DBORIGLINK_]
5415 vdb[DBSYNC_] = DBSYNCOK_
5416 vdb[DBDESC_] += u' OK. Created in DB.'
5417
5418 if doout:
5419 prt4_(u'\t\t*** makefile/sym DB(%s).' % (k,), place=u'f_syncsync')
5420 else:
5421
5422 se += 1
5423 vdb[DBSYNC_] |= DBSYNCERROTHER_
5424 vdb[DBDESC_] += u'Error in program. Unavailable operation in DIR for sync.'
5425 prt1_(u'<R>ERROR: Unavailable operation in DIR for sync DIR(%s).</R>' %
5426 (vdb_DBDIR_,), place=u'f_syncsync')
5427 continue
5428
5429
5430
5431 prt5_(u'f_syncsync: END synchronizing.')
5432 log_(u'end f_syncsync')
5433 return (sd, sn, sc, su, se, ts)
5434
5436 """
5437 Clear DB.
5438 Remove from DB unused record (successfully deleted and ignored items).
5439 """
5440 log(u'start f_syncclear')
5441 _db_ = cfg.Z_DB
5442 LOCK_ = cfg.LOCK
5443 LI_PERCENT_ = LI_PERCENT
5444 LI_CANCEL_ = LI_CANCEL
5445 prt3_ = prt3
5446 dirflags = DBDIRDELETE | DBDIRIGNORED
5447 dirskip = DBDIRUNCHANGED | DBDIRNEW
5448 DBDIRERROR_ = DBDIRERROR
5449 DBSYNCERROR_ = DBSYNCERROR
5450 DBSYNC_ = DBSYNC
5451 DBDIR_ = DBDIR
5452
5453 LOCK_[LI_PERCENT100] = len(_db_)
5454 LOCK_[LI_DESC] = u'Clear DB.'
5455 LOCK_[LI_PERCENT_] = 0
5456 if LOCK_[LI_CANCEL_]:
5457 prt3_(u'CANCELED.',place=u'f_syncclear')
5458 return
5459
5460 ks = cfg.Z_KS
5461 LOOPN_ = LOOPN
5462 LN = 0
5463 for k in ks:
5464 LN += 1
5465 if LN >= LOOPN_:
5466 LOCK_[LI_PERCENT_] += LN
5467 LN = 0
5468 if LOCK_[LI_CANCEL_]:
5469 prt3_(u'CANCELED.',place=u'f_syncclear')
5470 return
5471 vdb = _db_[k]
5472
5473
5474 if ( ( (vdb[DBDIR_] & dirflags) or ( (vdb[DBDIR_] & dirskip) == dirskip) )
5475 and not (vdb[DBDIR_] & DBDIRERROR_)
5476 and not (vdb[DBSYNC_] & DBSYNCERROR_) ):
5477 del cfg.Z_DB[k]
5478
5479
5480
5481
5482 LOCK_[LI_PERCENT_] += LN
5483 cfg.Z_KS = []
5484 log(u'end f_syncclear')
5485
5487 """
5488 Main function.
5489 """
5490 z_r.clear()
5491
5492 if len(cfg.arguments)>=2 and cfg.arguments[1] == u'--debug':
5493 del cfg.arguments[1]
5494 cfg.PYBAG_DEBUG = True
5495 cfg.PYBAG_LOG = True
5496 cfg.PYBAG_COLORIZE = True
5497 cfg.VERBOSE = 5
5498 _debug()
5499 else:
5500
5501 log(u'start MAIN')
5502 try:
5503 if len(cfg.arguments)<=1:
5504 cmd_gui(None)
5505 else:
5506 cfg.ISGUI = False
5507 pc = parse_command()
5508 if pc:
5509 if pc.cmd in [u'dist', u'version', u'help', u'usage',
5510 u'author', u'copyright', u'license', u'about', u'gui']:
5511 prt5(u'No init for command %s.' % (pc.cmd,),place=u'main')
5512 command(pc)
5513 else:
5514 if init_program():
5515 command(pc)
5516 except Exception, e:
5517 prt0( u'<r>EXCEPTION in programm: %s</r>' % (z_u(e),), place=u'main', error=e )
5518 log(u'end MAIN')
5519
5520
5521
5522
5523
5524
5525
5526
5527
5528
5529 -def _debug():
5530 """
5531 Debug function.
5532
5533 Test functions.
5534 """
5535 print(u'Start testing.\n')
5536 log(u'Start testing.\n')
5537 log()
5538
5539
5540
5541 global dir_test, dir_db, dir_files, dir_cfg, dir_sync
5542
5543 dir_test = os.path.join(dir_root, u'test')
5544 dir_db = os.path.join(dir_test, u'db.gz')
5545 dir_files = os.path.join(dir_test, u'bagfiles')
5546 dir_cfg = os.path.join(dir_test, u'cfg')
5547
5548 dir_sync = os.path.join(dir_test, u'syncfiles')
5549
5550 cfg.Z_CP.load()
5551
5552
5553 _all = 0
5554 fails = 0
5555 excs = 0
5556 if len(cfg.arguments)>=2:
5557 tc = [ u'_test_' + o for o in cfg.arguments[1:] ]
5558 else:
5559 tc = [ o for o in dir(_module) ]
5560
5561
5562
5563 for tf in tc:
5564
5565
5566 if tf.startswith(u'_test_') and hasattr(sys.modules[__name__], tf) and callable(getattr(sys.modules[__name__], tf)):
5567
5568 _all = _all + 1
5569 log(u'Testing %s ........ ' % (tf,))
5570 print(u'Testing %s ........ ' % (tf,))
5571 r= False
5572 try:
5573 r = getattr(sys.modules[__name__], tf)()
5574 pass
5575 except Exception,exc:
5576 excs = excs + 1
5577 log(u'\tEXCEPTION %s' % (str(exc),))
5578 print(u'\n\tEXCEPTION %s\n' % (str(exc),))
5579 if not r:
5580 fails = fails + 1
5581 r = u'FAIL'
5582 else:
5583 r = u'OK'
5584 log(u'\t\t ---> %s' % (r,))
5585 print(u'\t\t ---> %s' % (r,))
5586
5587
5588 print (u'All modules %d, fails %d, exceptions %d\n' % (_all, fails, excs) )
5589 log (u'All modules %d, fails %d, exceptions %d\n' % (_all, fails, excs) )
5590
5591 log()
5592 log(u'End testing.\n')
5593 print(u'End testing.\n')
5594
5601 """
5602 Only for test
5603 """
5604 return True
5605
5607 """
5608 Only for test
5609 """
5610 return False
5611
5613 rr = True
5614
5615 gp = z_gp_class()
5616
5617
5618 log(u'#1 ---')
5619 r = gp.getroots()
5620 r = not bool(r)
5621 rr = rr and r
5622 if not r:
5623 log(u'#1 get empty roots fail')
5624
5625
5626 log(u'#2 ---')
5627 r = gp.getopt(u'root', 'uignore', False)
5628 r = not bool(r)
5629 rr = rr and r
5630 if not r:
5631 log(u'#2 getopt root no create fail')
5632
5633
5634 log(u'#3 ---')
5635 r = gp.getopt(u'root', u'ignore', True)
5636 r = gp.has_section(u'root') and gp.has_option(u'root', u'ignore') and \
5637 (gp.get(u'root', u'ignore') == u'')
5638 rr = rr and r
5639 if not r:
5640 log(u'#3 getopt create root fail')
5641
5642
5643 log(u'#4 ---')
5644 ra0 = gp.addroot()
5645 r = (ra0==u'r0')
5646 rr = rr and r
5647 if not r:
5648 log(u'#4 getopt add root fail (ret="%s")' % (ra0,))
5649
5650
5651 log(u'#5 ---')
5652 gp.setopt(u'root', u'ignore', u'.svn')
5653 r = gp.getopt(u'root', u'ignore') == u'.svn'
5654 if not r:
5655 log(u'#5 setopt root fail')
5656 rr = rr and r
5657
5658
5659 log(u'#6 ---')
5660 r = gp.getopt(ra0, u'ignore') == u'.svn'
5661 if not r:
5662 log(u'#6 getopt for r0 fail')
5663 rr = rr and r
5664
5665
5666 log(u'#7 ---')
5667 gp.setopt(ra0, u'ignore', u'.cvs')
5668 r = gp.getopt(ra0, u'ignore') == u'.cvs'
5669 if not r:
5670 log(u'#7 setopt getopt r0 fail')
5671 rr = rr and r
5672
5673
5674 log(u'#8 ---')
5675 ra1 = gp.addroot()
5676 gp.setopt(ra1, u'compare', u'crc32')
5677 r = gp.getopt(ra1, 'compare') == 'crc32'
5678 rr = rr and r
5679 if not r:
5680 log(u'#8.1 getopt fail')
5681 r = gp.getopt(u'root', u'compare') == ''
5682 rr = rr and r
5683 if not r:
5684 log(u'#8.2 getopt fail')
5685
5686
5687 log(u'#9 ---')
5688 r = gp.getroots()
5689 r = (len(r)==2) and ((ra0 in r) and (ra1 in r))
5690 rr = rr and r
5691 if not r:
5692 log(u'#9 getroots fail')
5693
5694
5695 log(u'#10 ---')
5696 gp.setopt(u'r2', u'ignore', u'.cvs')
5697 r = not (u'r2' in gp.getroots())
5698 rr = rr and r
5699 if not r:
5700 log(u'#10 setopt setopt r2 fail')
5701
5702
5703 log(u'#11 ---')
5704 gp.delroot(u'r1')
5705 r = (not (u'r1' in gp.getroots())) and (len(gp.getroots()) == 1)
5706 rr = rr and r
5707 if not r:
5708 log(u'#11 setopt delroot r1 fail')
5709
5710
5711 log(u'#12 ---')
5712 ra2 = gp.addroot()
5713 gp.setopt(u'root', u'compare', u'nono')
5714 gp.setopt(ra2, u'compare', u'crc32')
5715 r = gp.getopt(ra2, u'compare') == u'crc32'
5716 rr = rr and r
5717 if not r:
5718 log(u'#12.1 getopt fail')
5719 r = gp.getopt(u'root', u'compare') == u'nono'
5720 rr = rr and r
5721 if not r:
5722 log(u'#12.2 getopt fail')
5723 gp.delopt(ra2, u'compare')
5724 r = gp.getopt(ra2, u'compare') == u'nono'
5725 if not r:
5726 log(u'#12.3 delopt fail')
5727
5728 return rr
5729
5732
5733
5734 db = {1:{DBDBCRC32:12345678, DBBAGSIZE:'bad type', DBBAGMDATE:'111'},2:{}}
5735
5736 log(u'#1 ---')
5737 dbg(dbdump(db))
5738 dbnormalize(db)
5739 dbg(dbdump(db))
5740 r = len(db) == 2
5741 r = r and len(db[1]) == len(z_db_formatlist)
5742 r = r and len(db[2]) == len(z_db_formatlist)
5743 if not r:
5744 log(u'#1 dbnormalize fail')
5745 tmp = { 1:{ DBBAGFLAG:DBFLAGNONE,
5746 DBBAGSIZE:DBNONE,
5747 DBBAGMDATE:111.0,
5748 DBBAGCRC32:DBNONE,
5749 DBBAGREAD:DBREADNONE,
5750 DBBAGPATH:DBPATHNONE,
5751 DBBAGLINK:DBLINKNONE,
5752 DBDBFLAG:DBFLAGNONE,
5753 DBDBSIZE:DBNONE,
5754 DBDBMDATE:DBMDATENONE,
5755 DBDBCRC32:12345678,
5756 DBDBREAD:DBREADNONE,
5757 DBDBPATH:DBPATHNONE,
5758 DBDBLINK:DBLINKNONE,
5759 DBORIGFLAG:DBFLAGNONE,
5760 DBORIGSIZE:DBNONE,
5761 DBORIGMDATE:DBMDATENONE,
5762 DBORIGCRC32:DBNONE,
5763 DBORIGREAD:DBREADNONE,
5764 DBORIGPATH:DBPATHNONE,
5765 DBORIGLINK:DBLINKNONE,
5766 DBDIR:DBDIRNONE,
5767 DBSYNC:DBSYNCNONE,
5768 DBCOMPARE:P_COMPARESTAT,
5769 DBSYMLINK:P_SYMLINKCOPY,
5770 DBDESC:DBDESCNONE,
5771 DBDIRMAN:DBDIRNONE,
5772 DBDESCMAN:DBDESCMANNONE,
5773 DBITEMID:DBITEMNONE,
5774 DBDIRREADED:DBDIRNONE,
5775 DBDESCREADED:DBDESCNONE,
5776 DBSYMLINKEMUL:DBSYMEMULNONE,
5777 DBBACKUP:DBBACKUPNO},
5778 2:{ DBBAGFLAG:DBFLAGNONE,
5779 DBBAGSIZE:DBNONE,
5780 DBBAGMDATE:DBMDATENONE,
5781 DBBAGCRC32:DBNONE,
5782 DBBAGREAD:DBREADNONE,
5783 DBBAGPATH:DBPATHNONE,
5784 DBBAGLINK:DBLINKNONE,
5785 DBDBFLAG:DBFLAGNONE,
5786 DBDBSIZE:DBNONE,
5787 DBDBMDATE:DBMDATENONE,
5788 DBDBCRC32:DBNONE,
5789 DBDBREAD:DBREADNONE,
5790 DBDBPATH:DBPATHNONE,
5791 DBDBLINK:DBLINKNONE,
5792 DBORIGFLAG:DBFLAGNONE,
5793 DBORIGSIZE:DBNONE,
5794 DBORIGMDATE:DBMDATENONE,
5795 DBORIGCRC32:DBNONE,
5796 DBORIGREAD:DBREADNONE,
5797 DBORIGPATH:DBPATHNONE,
5798 DBORIGLINK:DBLINKNONE,
5799 DBDIR:DBDIRNONE,
5800 DBSYNC:DBSYNCNONE,
5801 DBCOMPARE:P_COMPARESTAT,
5802 DBSYMLINK:P_SYMLINKCOPY,
5803 DBDESC:DBDESCNONE,
5804 DBDIRMAN:DBDIRNONE,
5805 DBDESCMAN:DBDESCMANNONE,
5806 DBITEMID:DBITEMNONE,
5807 DBDIRREADED:DBDIRNONE,
5808 DBDESCREADED:DBDESCNONE,
5809 DBSYMLINKEMUL:DBSYMEMULNONE,
5810 DBBACKUP:DBBACKUPNO}
5811 }
5812
5813
5814 dbg(dbdump(tmp))
5815 r = r and db == tmp
5816 if not r:
5817 log(u'#2 dbnormalize fail')
5818 return r
5819
5821 rr = True
5822
5823 p1 = 'http://www.qqq.ru/root/qwerty/poiuy/erty.yyy'
5824 r1 = ['http:\\', 'www.qqq.ru', 'root', 'qwerty', 'poiuy', 'erty.yyy']
5825 r2 = splitpaths(p1)
5826 r = r2 == r1
5827 rr = rr and r
5828 if not r:
5829 log(u'#1 splitpaths fail\npl="%s"\nrez="%s"\n' % (p1,r2))
5830
5831 p1 = 'erty.yyy'
5832 r1 = ['erty.yyy']
5833 r = splitpaths(p1)
5834 r = r == r1
5835 rr = rr and r
5836 if not r:
5837 log(u'#2 splitpaths fail')
5838
5839 if cfg.OSWIN:
5840 p1 = 'D:/root/qwerty/poiuy/erty.yyy'
5841 r1 = ['D:\\', 'root', 'qwerty', 'poiuy', 'erty.yyy']
5842 r2 = splitpaths(p1)
5843 r = r2 == r1
5844 rr = rr and r
5845 if not r:
5846 log(u'#3 splitpaths fail\npl="%s"\nrez="%s"\n' % (p1,r2))
5847
5848 if not cfg.OSWIN:
5849 p1 = '/root/qwerty/poiuy/erty.yyy'
5850 r1 = ['/', 'root', 'qwerty', 'poiuy', 'erty.yyy']
5851 r2 = splitpaths(p1)
5852 r = r2 == r1
5853 rr = rr and r
5854 if not r:
5855 log(u'#4 splitpaths fail\npl="%s"\nrez="%s"\n' % (p1,r2))
5856
5857 p1 = 'http:///www.qqq.ru/root/qwerty/poiuy//erty.yyy'
5858 r1 = ['http:\\', 'www.qqq.ru', 'root', 'qwerty', 'poiuy', 'erty.yyy']
5859 r2 = splitpaths(p1)
5860 r = r2 == r1
5861 rr = rr and r
5862 if not r:
5863 log(u'#5 splitpaths fail\npl="%s"\nrez="%s"\n' % (p1,r2))
5864
5865 return rr
5866
5867
5868
5869
5870
5871
5872
5873
5874
5875
5876
5877
5878
5879
5880
5881
5882
5883
5884
5885
5886
5887
5888
5889
5890
5891
5892
5893
5894
5895
5896
5897
5898
5899
5900
5901
5902
5903
5904
5905
5906
5907 -def _test_dbread():
5908 rr = True
5909 p1 = u'dir1'
5910 p2 = u'dir1/file1.txt'
5911 p1n = path2db(p1)
5912 p2n = path2db(p2)
5913 db = dbread()
5914
5915
5916
5917 t = {p1n: {0: p1n, 301: u'd', 302: u'0',
5918 303: u'1233061788.5309999', 304: u'0', 305: True, DBDBLINK:DBLINKNONE},
5919 p2n: {0: p2n, 301: u'f',
5920 302: u'16', 303: u'1233061988.5309999', 304: u'0',
5921 305: True, DBDBLINK:u'/link/point'}}
5922 dbg('db="%s"' % (str(db),))
5923 dbg('t="%s"' % (str(t),))
5924 r = db == t
5925 rr = rr and r
5926 if not r:
5927 log(u'#1 dbread fail')
5928
5929 return rr
5930
5932 rr = True
5933
5934 t = cfg.Z_CP.findroot('dir1')
5935 dbg(' findroot=', t)
5936 r = t == 'r10'
5937 rr = rr and r
5938
5939 if not r:
5940 log(u'#1 findroot fail')
5941
5942 t = cfg.Z_CP.findroot('file2.txt')
5943 r = t == 'r20'
5944 rr = rr and r
5945 if not r:
5946 log(u'#2 findroot fail')
5947
5948 t = cfg.Z_CP.findroot('dir1/file1.txt')
5949 r = t == 'r10'
5950 rr = rr and r
5951 if not r:
5952 log(u'#3 findroot fail')
5953
5954 t = cfg.Z_CP.findroot('dir222')
5955 r = t == None
5956 rr = rr and r
5957 if not r:
5958 log(u'#4 findroot fail')
5959
5960 return rr
5961
5963 rr = True
5964
5965 p = u'dir1'
5966 t = validatepaths(u'r10', p, os.path.join(dir_sync,p))
5967 pr = [u'r10', os.path.normpath(os.path.join(dir_files,p)),
5968 os.path.normpath(os.path.join(dir_sync,p)),
5969 path2db(p)]
5970 dbg(u' validatepaths=', t)
5971 dbg(u' pr=', pr)
5972 r = t == pr
5973 rr = rr and r
5974 if not r:
5975 log(u'#1 validatepaths fail')
5976
5977 p = os.path.join(u'dir1', u'file1.txt')
5978 t = validatepaths(u'r10', p, os.path.join(dir_sync,p))
5979 pr = [u'r10', os.path.normpath(os.path.join(dir_files,p)),
5980 os.path.normpath(os.path.join(dir_sync,p)),
5981 path2db(p)]
5982 dbg(u' validatepaths=', t)
5983 dbg(u' pr=', pr)
5984 r = t == pr
5985 rr = rr and r
5986 if not r:
5987 log(u'#2 validatepaths fail')
5988
5989 p = u'file2.txt'
5990 t = validatepaths(u'r20', p, os.path.join(dir_sync,p))
5991 pr = ['r20', os.path.normpath(os.path.join(dir_files,p)),
5992 os.path.normpath(os.path.join(dir_sync,p)),
5993 path2db(p)]
5994 dbg(u' validatepaths=', t)
5995 dbg(u' pr=', pr)
5996 r = t == pr
5997 rr = rr and r
5998 if not r:
5999 log(u'#3 validatepaths fail')
6000
6001 p = os.path.join(u'.', u'file2.txt')
6002 t = validatepaths(u'r20', p, os.path.join(dir_sync,p))
6003 pr = ['r20', os.path.normpath(os.path.join(dir_files,p)),
6004 os.path.normpath(os.path.join(dir_sync,p)),
6005 path2db(p)]
6006 dbg(u' validatepaths=', t)
6007 dbg(u' pr=', pr)
6008 r = t == pr
6009 rr = rr and r
6010 if not r:
6011 log(u'#4 validatepaths fail')
6012
6013 p = os.path.join(u'dir1', u'файл3.txt')
6014 t = validatepaths(u'r10', p, os.path.join(dir_sync,p))
6015 pr = [u'r10', os.path.normpath(os.path.join(dir_files,p)),
6016 os.path.normpath(os.path.join(dir_sync,p)),
6017 path2db(p)]
6018 dbg(u' validatepaths=', t)
6019 dbg(u' pr=', pr)
6020 r = t == pr
6021 rr = rr and r
6022 if not r:
6023 log(u'#2 validatepaths fail')
6024
6025 return rr
6026
6028 rr = True
6029 db = dbread()
6030 pb = u'dir1'
6031 ps = os.path.join(dir_sync,pb)
6032 r = cfg.Z_CP.findroot(pb)
6033 (r, pb1, ps1, i) = validatepaths(r, pb, ps)
6034 dbg('_test_dbwalkfiles: bag')
6035 dbwalkfiles(db,r,pb1, True, i)
6036 dbg('_test_dbwalkfiles: sync')
6037 dbwalkfiles(db,r,ps1, False, i)
6038 dbg('************* \n DB =')
6039 dbg(z_u(db))
6040 p1 = u'dir1'
6041 p2 = os.path.join(u'dir1', u'file1.txt')
6042 p3 = os.path.join(u'dir1', u'файл3.txt')
6043 p4 = os.path.join(u'dir1', u'ДИР1_1')
6044 t = { path2db(p1):{DBPATH:path2db(p1),
6045 DBDBFLAG:DBFLAGNONE, DBDBSIZE:DBNONE, DBDBMDATE:DBMDATENONE,
6046 DBDBCRC32:DBNONE, DBDBREAD:DBREADNONE, DBDBPATH:path2os(path2db(p1)),
6047 DBBAGFLAG:DBFLAGDIR, DBBAGSIZE:DBNONE,
6048 DBBAGMDATE:1233380243.53125, DBSYMLINKEMUL:DBSYMEMULNONE},
6049 path2db(p2):{DBPATH:path2db(p2),
6050 },
6051 path2db(p3):{DBPATH:path2db(p3),
6052 },
6053 path2db(p4):{DBPATH:path2db(p4),
6054 }
6055 }
6056
6057
6058 prt3(u'WARNING. dbwalkfiles not real tested.', place=u'dbwalkfiles')
6059 r = True
6060 rr = rr and r
6061 if not r:
6062 log(u'#1 validatepaths fail')
6063
6064 return rr
6065
6067 rr = True
6068 class cmd(object):
6069 pass
6070
6071 r = transopt(u'u',cmd)
6072 r = r == u''
6073 rr = rr and r
6074 if not r:
6075 log(u'#1 fail')
6076
6077 r = transopt(u'k',cmd)
6078 r = r == P_DEFSYMLINK
6079 rr = rr and r
6080 if not r:
6081 log(u'#2 fail')
6082
6083 cmd.k='i'
6084 r = transopt(u'k',cmd)
6085 r = r == P_SYMLINKIGNORE
6086 rr = rr and r
6087 if not r:
6088 log(u'#3 fail')
6089
6090 return rr
6091
6093 rr = True
6094
6095 p = u'dir1'
6096 t = u'dir1'
6097 d = path2db(p)
6098 dbg( u'_test_path2db: p="%s", t="%s", d="%s".' % (p,t,d) )
6099 r = t == d
6100 rr = rr and r
6101 if not r:
6102 log(u'#1 path2db fail')
6103
6104 p = u'dir1/файл.txt'
6105 t = u'dir1/файл.txt'
6106 d = path2db(p)
6107 dbg( u'_test_path2db: p="%s", t="%s", d="%s".' % (p,t,d) )
6108 r = t == d
6109 rr = rr and r
6110 if not r:
6111 log(u'#2 path2db fail')
6112
6113 p = u'dir1/файл 1.txt'
6114 t = u'dir1/файл'+DBESCESC+u'00201.txt'
6115 d = path2db(p)
6116 dbg( u'_test_path2db: p="%s", t="%s", d="%s".' % (p,t,d) )
6117 r = t == d
6118 rr = rr and r
6119 if not r:
6120 log(u'#3 path2db fail')
6121
6122
6123
6124
6125
6126
6127
6128
6129
6130
6131
6132
6133 p = u'\\'+DBESCESC+u'/dir1/файл 1.txt'
6134 t = u'\\/'+DBESCESC+u'0023/dir1/файл'+DBESCESC+u'00201.txt'
6135 d = path2db(p)
6136 dbg( u'_test_path2db: p="%s", t="%s", d="%s".' % (p,t,d) )
6137 r = t == d
6138 rr = rr and r
6139 if not r:
6140 log(u'#5 path2db fail')
6141
6142 return rr
6143
6145 rr = True
6146
6147 t = u'dir1'
6148 p = u'dir1'
6149 d = path2os(p)
6150 dbg( u'_test_path2os: p="%s", t="%s", d="%s".' % (p,t,d) )
6151 r = t == d
6152 rr = rr and r
6153 if not r:
6154 log(u'#1 path2db fail')
6155
6156 p = u'dir1/файл.txt'
6157 t = os.path.join(u'dir1', u'файл.txt')
6158 d = path2os(p)
6159 dbg( u'_test_path2os: p="%s", t="%s", d="%s".' % (p,t,d) )
6160 r = t == d
6161 rr = rr and r
6162 if not r:
6163 log(u'#2 path2os fail')
6164
6165 t = os.path.join(u'dir1', u'файл 1.txt')
6166 p = u'dir1/файл'+DBESCESC+u'00201.txt'
6167 d = path2os(p)
6168 dbg( u'_test_path2db: p="%s", t="%s", d="%s".' % (p,t,d) )
6169 r = t == d
6170 rr = rr and r
6171 if not r:
6172 log(u'#3 path2os fail')
6173
6174 t = os.path.join(u'/', DBESCESC, u'dir1', u'файл 1.txt')
6175 p = DBESCESC+u'002F/'+DBESCESC+u'0023/dir1/файл'+DBESCESC+u'00201.txt'
6176 d = path2os(p)
6177 dbg( u'_test_path2db: p="%s", t="%s", d="%s".' % (p,t,d) )
6178 r = t == d
6179 rr = rr and r
6180 if not r:
6181 log(u'#4 path2os fail')
6182
6183 t = os.path.join(u'\\',DBESCESC,u'dir1',u'файл 1.txt')
6184 p = u'\\'+DBESCESC+u'0023/dir1/файл'+DBESCESC+u'00201.txt'
6185 d = path2os(p)
6186 dbg( u'_test_path2db: p="%s", t="%s", d="%s".' % (p,t,d) )
6187 r = t == d
6188 rr = rr and r
6189 if not r:
6190 log(u'#5 path2os fail')
6191
6192 t = os.path.join(u'e:', u'dir1', u'файл 1.txt')
6193 p = u'e:/dir1/файл'+DBESCESC+u'00201.txt'
6194 d = path2os(p)
6195 dbg( u'_test_path2db: p="%s", t="%s", d="%s".' % (p,t,d) )
6196 r = t == d
6197 rr = rr and r
6198 if not r:
6199 log(u'#6 path2os fail')
6200
6201 return rr
6202
6204 rr = True
6205 t = u'/mount/flash/point/file.txt'
6206 p = os.path.join(dir_test, u'testlinkread.link')
6207 d = readlink(p)
6208 dbg( u'_test_readlink: p="%s", t="%s", d="%s".' % (p,t,d) )
6209 r = t == d
6210 rr = rr and r
6211 if not r:
6212 log(u'#1 readlink fail')
6213
6214 return rr
6215
6217 rr = True
6218 t = u'/mount/flash/point/file.txt'
6219 p = os.path.join(dir_test, u'testlinkwrite.link')
6220 try:
6221 if os.path.lexists(p):
6222 os.remove(p)
6223 except Exception, exc:
6224 raise Exception, u'Can not _test_symlink, can not remove existing. "%s"' % (z_u(exc),)
6225 d = symlink(t,p)
6226 if symlinktest(p):
6227 s = readlink(p)
6228 dbg( u'_test_symlink: readed real="%s".' % (s,) )
6229 r = s == t
6230 else:
6231 fp = codecs.open(p, 'r', 'utf_8', 'strict', 0)
6232 s = fp.read()
6233 dbg( u'_test_symlink: readed emulated="%s".' % (s,) )
6234 r = s == u'%s%s%s' % (u'pybaglink start:', t, u':pybaglink end')
6235 rr = rr and r
6236 if not r:
6237 log(u'#1 symlink fail')
6238
6239 return rr
6240
6242 rr = True
6243 t = -1842165055
6244 p = os.path.join(dir_sync, u'file2.txt')
6245 d = getcrc32(p)
6246 dbg(u'_test_getcrc32: CRC=%d.' % (d,))
6247 r = d == t
6248 rr = rr and r
6249 if not r:
6250 log(u'#1 symlink fail')
6251
6252 return rr
6253
6255 rr = True
6256 err = u'Option string for report format is wrong.'
6257 opt = REPORTFILTER_DEFAULT
6258 t = (0,
6259 u'!.*',
6260 u'fdsu',
6261 DBDIRBAG | DBDIRORIG | DBDIRNEW | DBDIRCOPY | DBDIRFORCED | \
6262 DBDIRDELETE | DBDIRTS | DBDIRWARNING | DBDIRCONFLICT | DBDIRERROR,
6263 O_REPORTFORMAT[u's'])
6264 terr = u''
6265 rerr = transreport(opt)
6266 r = (cfg.REPORTPAGE, cfg.REPORTPATT, cfg.REPORTTYPE, cfg.REPORTFILTER, cfg.REPORTFORMAT)
6267 dbg(u'r = ', str(r))
6268 dbg(u't = ', str(t))
6269 dbg(u'rerr = ', str(rerr))
6270 dbg(u'terr = ', str(terr))
6271 r = (r == t) and (rerr == terr)
6272 rr = rr and r
6273 if not r:
6274 log(u'#1 transreport fail')
6275
6276 opt = u'lz37bonhe1r--*\\*.log--m..%(dir)s..'
6277 t = (37,
6278 u'*\\*.log',
6279 u'f',
6280 DBDIRBAG | DBDIRORIG | DBDIRNEW | DBDIRCOPY | DBDIRERROR,
6281 O_REPORTFORMAT[u'l'])
6282 terr = u''
6283 rerr = transreport(opt)
6284 r = (cfg.REPORTPAGE, cfg.REPORTPATT, cfg.REPORTTYPE, cfg.REPORTFILTER, cfg.REPORTFORMAT)
6285 dbg(u'r = ', str(r))
6286 dbg(u't = ', str(t))
6287 dbg(u'rerr = ', str(rerr))
6288 dbg(u'terr = ', str(terr))
6289 r = (r == t) and (rerr == terr)
6290 rr = rr and r
6291 if not r:
6292 log(u'#2 transreport fail')
6293
6294 opt = u'm.. %(forced)s%(dir)s%(err)s [%(path)s]\n\t%(desc)s..z37bonhe1r--*\\*.log--2'
6295 t = (37,
6296 u'*\\*.log',
6297 u'd',
6298 DBDIRBAG | DBDIRORIG | DBDIRNEW | DBDIRCOPY | DBDIRERROR,
6299 u' %(forced)s%(dir)s%(err)s [%(path)s]\n\t%(desc)s')
6300 terr = u''
6301 rerr = transreport(opt)
6302 r = (cfg.REPORTPAGE, cfg.REPORTPATT, cfg.REPORTTYPE, cfg.REPORTFILTER, cfg.REPORTFORMAT)
6303 dbg(u'r = ', str(r))
6304 dbg(u't = ', str(t))
6305 dbg(u'rerr = ', str(rerr))
6306 dbg(u'terr = ', str(terr))
6307 r = (r == t) and (rerr == terr)
6308 rr = rr and r
6309 if not r:
6310 log(u'#3 transreport fail')
6311
6312 opt = u'Gm.. %(forced)s%(dir)s%(err)s [%(path)s]\n\t%(desc)s..z37bonhe1r--*\\*.log--2'
6313 t = (37,
6314 u'*\\*.log',
6315 u'd',
6316 DBDIRBAG | DBDIRORIG | DBDIRNEW | DBDIRCOPY | DBDIRERROR,
6317 u' %(forced)s%(dir)s%(err)s [%(path)s]\n\t%(desc)s')
6318 terr = err
6319 rerr = transreport(opt)
6320
6321 dbg(u'r = ', str(r))
6322 dbg(u't = ', str(t))
6323 dbg(u'rerr = ', str(rerr))
6324 dbg(u'terr = ', str(terr))
6325 r = rerr == terr
6326 rr = rr and r
6327 if not r:
6328 log(u'#4 transreport fail')
6329
6330 return rr
6331
6332
6333
6334
6335
6336 stdout_wrapper = termcolors.StreamProxy(None, True)
6337 stdout_wrapper.wrap_stdout()
6338
6339 stdout_wrapper.term_colors.reset()
6340
6341
6342 -def mainstart():
6343 exc = None
6344 try:
6345
6346 atexit.register(deinit_program)
6347 init_dirs()
6348 init_encoding()
6349 log(u'PYBAG started at ', time.ctime())
6350 log(u'Python version: %s' % sys.version)
6351 log(u'dir_root="%s"' % dir_root)
6352 log(u'dir_db="%s"' % dir_db)
6353 log(u'dir_cfg="%s"' % dir_cfg)
6354 log(u'dir_files="%s"' % dir_files)
6355 cfg.Z_CP = z_gp_class()
6356 main()
6357 except Exception, exc:
6358 try:
6359 log(u'ERROR in program', error=exc, place=u'mainstart')
6360 except:
6361 pass
6362 try:
6363 log(u'PYBAG ended at ', time.ctime())
6364 except:
6365 pass
6366