filecmp.py 9.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309
  1. """Utilities for comparing files and directories.
  2. Classes:
  3. dircmp
  4. Functions:
  5. cmp(f1, f2, shallow=True) -> int
  6. cmpfiles(a, b, common) -> ([], [], [])
  7. clear_cache()
  8. """
  9. import os
  10. import stat
  11. from itertools import filterfalse
  12. from types import GenericAlias
  13. __all__ = ['clear_cache', 'cmp', 'dircmp', 'cmpfiles', 'DEFAULT_IGNORES']
  14. _cache = {}
  15. BUFSIZE = 8*1024
  16. DEFAULT_IGNORES = [
  17. 'RCS', 'CVS', 'tags', '.git', '.hg', '.bzr', '_darcs', '__pycache__']
  18. def clear_cache():
  19. """Clear the filecmp cache."""
  20. _cache.clear()
  21. def cmp(f1, f2, shallow=True):
  22. """Compare two files.
  23. Arguments:
  24. f1 -- First file name
  25. f2 -- Second file name
  26. shallow -- Just check stat signature (do not read the files).
  27. defaults to True.
  28. Return value:
  29. True if the files are the same, False otherwise.
  30. This function uses a cache for past comparisons and the results,
  31. with cache entries invalidated if their stat information
  32. changes. The cache may be cleared by calling clear_cache().
  33. """
  34. s1 = _sig(os.stat(f1))
  35. s2 = _sig(os.stat(f2))
  36. if s1[0] != stat.S_IFREG or s2[0] != stat.S_IFREG:
  37. return False
  38. if shallow and s1 == s2:
  39. return True
  40. if s1[1] != s2[1]:
  41. return False
  42. outcome = _cache.get((f1, f2, s1, s2))
  43. if outcome is None:
  44. outcome = _do_cmp(f1, f2)
  45. if len(_cache) > 100: # limit the maximum size of the cache
  46. clear_cache()
  47. _cache[f1, f2, s1, s2] = outcome
  48. return outcome
  49. def _sig(st):
  50. return (stat.S_IFMT(st.st_mode),
  51. st.st_size,
  52. st.st_mtime)
  53. def _do_cmp(f1, f2):
  54. bufsize = BUFSIZE
  55. with open(f1, 'rb') as fp1, open(f2, 'rb') as fp2:
  56. while True:
  57. b1 = fp1.read(bufsize)
  58. b2 = fp2.read(bufsize)
  59. if b1 != b2:
  60. return False
  61. if not b1:
  62. return True
  63. # Directory comparison class.
  64. #
  65. class dircmp:
  66. """A class that manages the comparison of 2 directories.
  67. dircmp(a, b, ignore=None, hide=None)
  68. A and B are directories.
  69. IGNORE is a list of names to ignore,
  70. defaults to DEFAULT_IGNORES.
  71. HIDE is a list of names to hide,
  72. defaults to [os.curdir, os.pardir].
  73. High level usage:
  74. x = dircmp(dir1, dir2)
  75. x.report() -> prints a report on the differences between dir1 and dir2
  76. or
  77. x.report_partial_closure() -> prints report on differences between dir1
  78. and dir2, and reports on common immediate subdirectories.
  79. x.report_full_closure() -> like report_partial_closure,
  80. but fully recursive.
  81. Attributes:
  82. left_list, right_list: The files in dir1 and dir2,
  83. filtered by hide and ignore.
  84. common: a list of names in both dir1 and dir2.
  85. left_only, right_only: names only in dir1, dir2.
  86. common_dirs: subdirectories in both dir1 and dir2.
  87. common_files: files in both dir1 and dir2.
  88. common_funny: names in both dir1 and dir2 where the type differs between
  89. dir1 and dir2, or the name is not stat-able.
  90. same_files: list of identical files.
  91. diff_files: list of filenames which differ.
  92. funny_files: list of files which could not be compared.
  93. subdirs: a dictionary of dircmp objects, keyed by names in common_dirs.
  94. """
  95. def __init__(self, a, b, ignore=None, hide=None): # Initialize
  96. self.left = a
  97. self.right = b
  98. if hide is None:
  99. self.hide = [os.curdir, os.pardir] # Names never to be shown
  100. else:
  101. self.hide = hide
  102. if ignore is None:
  103. self.ignore = DEFAULT_IGNORES
  104. else:
  105. self.ignore = ignore
  106. def phase0(self): # Compare everything except common subdirectories
  107. self.left_list = _filter(os.listdir(self.left),
  108. self.hide+self.ignore)
  109. self.right_list = _filter(os.listdir(self.right),
  110. self.hide+self.ignore)
  111. self.left_list.sort()
  112. self.right_list.sort()
  113. def phase1(self): # Compute common names
  114. a = dict(zip(map(os.path.normcase, self.left_list), self.left_list))
  115. b = dict(zip(map(os.path.normcase, self.right_list), self.right_list))
  116. self.common = list(map(a.__getitem__, filter(b.__contains__, a)))
  117. self.left_only = list(map(a.__getitem__, filterfalse(b.__contains__, a)))
  118. self.right_only = list(map(b.__getitem__, filterfalse(a.__contains__, b)))
  119. def phase2(self): # Distinguish files, directories, funnies
  120. self.common_dirs = []
  121. self.common_files = []
  122. self.common_funny = []
  123. for x in self.common:
  124. a_path = os.path.join(self.left, x)
  125. b_path = os.path.join(self.right, x)
  126. ok = 1
  127. try:
  128. a_stat = os.stat(a_path)
  129. except OSError:
  130. # print('Can\'t stat', a_path, ':', why.args[1])
  131. ok = 0
  132. try:
  133. b_stat = os.stat(b_path)
  134. except OSError:
  135. # print('Can\'t stat', b_path, ':', why.args[1])
  136. ok = 0
  137. if ok:
  138. a_type = stat.S_IFMT(a_stat.st_mode)
  139. b_type = stat.S_IFMT(b_stat.st_mode)
  140. if a_type != b_type:
  141. self.common_funny.append(x)
  142. elif stat.S_ISDIR(a_type):
  143. self.common_dirs.append(x)
  144. elif stat.S_ISREG(a_type):
  145. self.common_files.append(x)
  146. else:
  147. self.common_funny.append(x)
  148. else:
  149. self.common_funny.append(x)
  150. def phase3(self): # Find out differences between common files
  151. xx = cmpfiles(self.left, self.right, self.common_files)
  152. self.same_files, self.diff_files, self.funny_files = xx
  153. def phase4(self): # Find out differences between common subdirectories
  154. # A new dircmp object is created for each common subdirectory,
  155. # these are stored in a dictionary indexed by filename.
  156. # The hide and ignore properties are inherited from the parent
  157. self.subdirs = {}
  158. for x in self.common_dirs:
  159. a_x = os.path.join(self.left, x)
  160. b_x = os.path.join(self.right, x)
  161. self.subdirs[x] = dircmp(a_x, b_x, self.ignore, self.hide)
  162. def phase4_closure(self): # Recursively call phase4() on subdirectories
  163. self.phase4()
  164. for sd in self.subdirs.values():
  165. sd.phase4_closure()
  166. def report(self): # Print a report on the differences between a and b
  167. # Output format is purposely lousy
  168. print('diff', self.left, self.right)
  169. if self.left_only:
  170. self.left_only.sort()
  171. print('Only in', self.left, ':', self.left_only)
  172. if self.right_only:
  173. self.right_only.sort()
  174. print('Only in', self.right, ':', self.right_only)
  175. if self.same_files:
  176. self.same_files.sort()
  177. print('Identical files :', self.same_files)
  178. if self.diff_files:
  179. self.diff_files.sort()
  180. print('Differing files :', self.diff_files)
  181. if self.funny_files:
  182. self.funny_files.sort()
  183. print('Trouble with common files :', self.funny_files)
  184. if self.common_dirs:
  185. self.common_dirs.sort()
  186. print('Common subdirectories :', self.common_dirs)
  187. if self.common_funny:
  188. self.common_funny.sort()
  189. print('Common funny cases :', self.common_funny)
  190. def report_partial_closure(self): # Print reports on self and on subdirs
  191. self.report()
  192. for sd in self.subdirs.values():
  193. print()
  194. sd.report()
  195. def report_full_closure(self): # Report on self and subdirs recursively
  196. self.report()
  197. for sd in self.subdirs.values():
  198. print()
  199. sd.report_full_closure()
  200. methodmap = dict(subdirs=phase4,
  201. same_files=phase3, diff_files=phase3, funny_files=phase3,
  202. common_dirs = phase2, common_files=phase2, common_funny=phase2,
  203. common=phase1, left_only=phase1, right_only=phase1,
  204. left_list=phase0, right_list=phase0)
  205. def __getattr__(self, attr):
  206. if attr not in self.methodmap:
  207. raise AttributeError(attr)
  208. self.methodmap[attr](self)
  209. return getattr(self, attr)
  210. __class_getitem__ = classmethod(GenericAlias)
  211. def cmpfiles(a, b, common, shallow=True):
  212. """Compare common files in two directories.
  213. a, b -- directory names
  214. common -- list of file names found in both directories
  215. shallow -- if true, do comparison based solely on stat() information
  216. Returns a tuple of three lists:
  217. files that compare equal
  218. files that are different
  219. filenames that aren't regular files.
  220. """
  221. res = ([], [], [])
  222. for x in common:
  223. ax = os.path.join(a, x)
  224. bx = os.path.join(b, x)
  225. res[_cmp(ax, bx, shallow)].append(x)
  226. return res
  227. # Compare two files.
  228. # Return:
  229. # 0 for equal
  230. # 1 for different
  231. # 2 for funny cases (can't stat, etc.)
  232. #
  233. def _cmp(a, b, sh, abs=abs, cmp=cmp):
  234. try:
  235. return not abs(cmp(a, b, sh))
  236. except OSError:
  237. return 2
  238. # Return a copy with items that occur in skip removed.
  239. #
  240. def _filter(flist, skip):
  241. return list(filterfalse(skip.__contains__, flist))
  242. # Demonstration and testing.
  243. #
  244. def demo():
  245. import sys
  246. import getopt
  247. options, args = getopt.getopt(sys.argv[1:], 'r')
  248. if len(args) != 2:
  249. raise getopt.GetoptError('need exactly two args', None)
  250. dd = dircmp(args[0], args[1])
  251. if ('-r', '') in options:
  252. dd.report_full_closure()
  253. else:
  254. dd.report()
  255. if __name__ == '__main__':
  256. demo()