src/black/cache.py
  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
---- ---- ---- ---- ---- ---- ---- -- -- --- --- --- --- --- --- ------- ------- ------- | """Caching of formatted files with feature-based invalidation."""

---- ---- ---- ---- ---- ---- ---- -- -- --- --- --- --- --- --- ------- ------- ------- | 

---- ---- ---- ---- ---- ---- ---- -- -- --- --- --- --- --- --- ------- ------- ------- | import hashlib

---- ---- ---- ---- ---- ---- ---- -- -- --- --- --- --- --- --- ------- ------- ------- | import os

---- ---- ---- ---- ---- ---- ---- -- -- --- --- --- --- --- --- ------- ------- ------- | import pickle

---- ---- ---- ---- ---- ---- ---- -- -- --- --- --- --- --- --- ------- ------- ------- | import sys

---- ---- ---- ---- ---- ---- ---- -- -- --- --- --- --- --- --- ------- ------- ------- | import tempfile

---- ---- ---- ---- ---- ---- ---- -- -- --- --- --- --- --- --- ------- ------- ------- | from dataclasses import dataclass, field

---- ---- ---- ---- ---- ---- ---- -- -- --- --- --- --- --- --- ------- ------- ------- | from pathlib import Path

---- ---- ---- ---- ---- ---- ---- -- -- --- --- --- --- --- --- ------- ------- ------- | from typing import Dict, Iterable, NamedTuple, Set, Tuple

---- ---- ---- ---- ---- ---- ---- -- -- --- --- --- --- --- --- ------- ------- ------- | 

---- ---- ---- ---- ---- ---- ---- -- -- --- --- --- --- --- --- ------- ------- ------- | from platformdirs import user_cache_dir

---- ---- ---- ---- ---- ---- ---- -- -- --- --- --- --- --- --- ------- ------- ------- | 

---- ---- ---- ---- ---- ---- ---- -- -- --- --- --- --- --- --- ------- ------- ------- | from _black_version import version as __version__

---- ---- ---- ---- ---- ---- ---- -- -- --- --- --- --- --- --- ------- ------- ------- | from black.mode import Mode

---- ---- ---- ---- ---- ---- ---- -- -- --- --- --- --- --- --- ------- ------- ------- | 

---- ---- ---- ---- ---- ---- ---- -- -- --- --- --- --- --- --- ------- ------- ------- | if sys.version_info >= (3, 11):

---- ---- ---- ---- ---- ---- ---- -- -- --- --- --- --- --- --- ------- ------- ------- |     from typing import Self

---- ---- ---- ---- ---- ---- ---- -- -- --- --- --- --- --- --- ------- ------- ------- | else:

---- ---- ---- ---- ---- ---- ---- -- -- --- --- --- --- --- --- ------- ------- ------- |     from typing_extensions import Self

---- ---- ---- ---- ---- ---- ---- -- -- --- --- --- --- --- --- ------- ------- ------- | 

---- ---- ---- ---- ---- ---- ---- -- -- --- --- --- --- --- --- ------- ------- ------- | 

---- ---- ---- ---- ---- ---- ---- 01 -- --- --- --- --- --- --- ------- ------- ------- | class FileData(NamedTuple):

---- ---- ---- ---- ---- ---- ---- 01 -- --- --- --- --- --- --- ------- ------- ------- |     st_mtime: float

---- ---- ---- ---- ---- ---- ---- 01 -- --- --- --- --- --- --- ------- ------- ------- |     st_size: int

---- ---- ---- ---- ---- ---- ---- 01 -- --- --- --- --- --- --- ------- ------- ------- |     hash: str

---- ---- ---- ---- ---- ---- ---- -- -- --- --- --- --- --- --- ------- ------- ------- | 

---- ---- ---- ---- ---- ---- ---- -- -- --- --- --- --- --- --- ------- ------- ------- | 

0015 0006 0005 0001 0007 0002 0001 -- 01 001 002 001 002 003 003 0004.75 0002.38 0000.50 | def get_cache_dir() -> Path:

0015 0006 0005 0001 0007 0002 0001 -- 01 001 002 001 002 003 003 0004.75 0002.38 0000.50 |     """Get the cache directory used by black.

0015 0006 0005 0001 0007 0002 0001 -- 01 001 002 001 002 003 003 0004.75 0002.38 0000.50 | 

0015 0006 0005 0001 0007 0002 0001 -- 01 001 002 001 002 003 003 0004.75 0002.38 0000.50 |     Users can customize this directory on all systems using `BLACK_CACHE_DIR`

0015 0006 0005 0001 0007 0002 0001 -- 01 001 002 001 002 003 003 0004.75 0002.38 0000.50 |     environment variable. By default, the cache directory is the user cache directory

0015 0006 0005 0001 0007 0002 0001 -- 01 001 002 001 002 003 003 0004.75 0002.38 0000.50 |     under the black application.

0015 0006 0005 0001 0007 0002 0001 -- 01 001 002 001 002 003 003 0004.75 0002.38 0000.50 | 

0015 0006 0005 0001 0007 0002 0001 -- 01 001 002 001 002 003 003 0004.75 0002.38 0000.50 |     This result is immediately set to a constant `black.cache.CACHE_DIR` as to avoid

0015 0006 0005 0001 0007 0002 0001 -- 01 001 002 001 002 003 003 0004.75 0002.38 0000.50 |     repeated calls.

0015 0006 0005 0001 0007 0002 0001 -- 01 001 002 001 002 003 003 0004.75 0002.38 0000.50 |     """

0015 0006 0005 0001 0007 0002 0001 -- 01 001 002 001 002 003 003 0004.75 0002.38 0000.50 |     # NOTE: Function mostly exists as a clean way to test getting the cache directory.

0015 0006 0005 0001 0007 0002 0001 -- 01 001 002 001 002 003 003 0004.75 0002.38 0000.50 |     default_cache_dir = user_cache_dir("black")

0015 0006 0005 0001 0007 0002 0001 -- 01 001 002 001 002 003 003 0004.75 0002.38 0000.50 |     cache_dir = Path(os.environ.get("BLACK_CACHE_DIR", default_cache_dir))

0015 0006 0005 0001 0007 0002 0001 -- 01 001 002 001 002 003 003 0004.75 0002.38 0000.50 |     cache_dir = cache_dir / __version__

0015 0006 0005 0001 0007 0002 0001 -- 01 001 002 001 002 003 003 0004.75 0002.38 0000.50 |     return cache_dir

---- ---- ---- ---- ---- ---- ---- -- -- --- --- --- --- --- --- ------- ------- ------- | 

---- ---- ---- ---- ---- ---- ---- -- -- --- --- --- --- --- --- ------- ------- ------- | 

---- ---- ---- ---- ---- ---- ---- -- -- --- --- --- --- --- --- ------- ------- ------- | CACHE_DIR = get_cache_dir()

---- ---- ---- ---- ---- ---- ---- -- -- --- --- --- --- --- --- ------- ------- ------- | 

---- ---- ---- ---- ---- ---- ---- -- -- --- --- --- --- --- --- ------- ------- ------- | 

0002 0002 0002 0000 0000 0000 0000 -- 01 001 002 001 002 003 003 0004.75 0002.38 0000.50 | def get_cache_file(mode: Mode) -> Path:

0002 0002 0002 0000 0000 0000 0000 -- 01 001 002 001 002 003 003 0004.75 0002.38 0000.50 |     return CACHE_DIR / f"cache.{mode.get_cache_key()}.pickle"

---- ---- ---- ---- ---- ---- ---- -- -- --- --- --- --- --- --- ------- ------- ------- | 

---- ---- ---- ---- ---- ---- ---- -- -- --- --- --- --- --- --- ------- ------- ------- | 

---- ---- ---- ---- ---- ---- ---- -- -- --- --- --- --- --- --- ------- ------- ------- | @dataclass

---- ---- ---- ---- ---- ---- ---- 04 -- --- --- --- --- --- --- ------- ------- ------- | class Cache:

---- ---- ---- ---- ---- ---- ---- 04 -- --- --- --- --- --- --- ------- ------- ------- |     mode: Mode

---- ---- ---- ---- ---- ---- ---- 04 -- --- --- --- --- --- --- ------- ------- ------- |     cache_file: Path

---- ---- ---- ---- ---- ---- ---- 04 -- --- --- --- --- --- --- ------- ------- ------- |     file_data: Dict[str, FileData] = field(default_factory=dict)

---- ---- ---- ---- ---- ---- ---- 04 -- --- --- --- --- --- --- ------- ------- ------- | 

---- ---- ---- ---- ---- ---- ---- 04 -- --- --- --- --- --- --- ------- ------- ------- |     @classmethod

0019 0015 0012 0000 0004 0003 0000 04 04 001 001 001 001 002 002 0002.00 0001.00 0000.50 |     def read(cls, mode: Mode) -> Self:

0019 0015 0012 0000 0004 0003 0000 04 04 001 001 001 001 002 002 0002.00 0001.00 0000.50 |         """Read the cache if it exists and is well-formed.

0019 0015 0012 0000 0004 0003 0000 04 04 001 001 001 001 002 002 0002.00 0001.00 0000.50 | 

0019 0015 0012 0000 0004 0003 0000 04 04 001 001 001 001 002 002 0002.00 0001.00 0000.50 |         If it is not well-formed, the call to write later should

0019 0015 0012 0000 0004 0003 0000 04 04 001 001 001 001 002 002 0002.00 0001.00 0000.50 |         resolve the issue.

0019 0015 0012 0000 0004 0003 0000 04 04 001 001 001 001 002 002 0002.00 0001.00 0000.50 |         """

0019 0015 0012 0000 0004 0003 0000 04 04 001 001 001 001 002 002 0002.00 0001.00 0000.50 |         cache_file = get_cache_file(mode)

0019 0015 0012 0000 0004 0003 0000 04 04 001 001 001 001 002 002 0002.00 0001.00 0000.50 |         if not cache_file.exists():

0019 0015 0012 0000 0004 0003 0000 04 04 001 001 001 001 002 002 0002.00 0001.00 0000.50 |             return cls(mode, cache_file)

0019 0015 0012 0000 0004 0003 0000 04 04 001 001 001 001 002 002 0002.00 0001.00 0000.50 | 

0019 0015 0012 0000 0004 0003 0000 04 04 001 001 001 001 002 002 0002.00 0001.00 0000.50 |         with cache_file.open("rb") as fobj:

0019 0015 0012 0000 0004 0003 0000 04 04 001 001 001 001 002 002 0002.00 0001.00 0000.50 |             try:

0019 0015 0012 0000 0004 0003 0000 04 04 001 001 001 001 002 002 0002.00 0001.00 0000.50 |                 data: Dict[str, Tuple[float, int, str]] = pickle.load(fobj)

0019 0015 0012 0000 0004 0003 0000 04 04 001 001 001 001 002 002 0002.00 0001.00 0000.50 |                 file_data = {k: FileData(*v) for k, v in data.items()}

0019 0015 0012 0000 0004 0003 0000 04 04 001 001 001 001 002 002 0002.00 0001.00 0000.50 |             except (pickle.UnpicklingError, ValueError, IndexError):

0019 0015 0012 0000 0004 0003 0000 04 04 001 001 001 001 002 002 0002.00 0001.00 0000.50 |                 return cls(mode, cache_file)

0019 0015 0012 0000 0004 0003 0000 04 04 001 001 001 001 002 002 0002.00 0001.00 0000.50 | 

0019 0015 0012 0000 0004 0003 0000 04 04 001 001 001 001 002 002 0002.00 0001.00 0000.50 |         return cls(mode, cache_file, file_data)

0019 0015 0012 0000 0004 0003 0000 04 -- --- --- --- --- --- --- ------- ------- ------- | 

---- ---- ---- ---- ---- ---- ---- 04 -- --- --- --- --- --- --- ------- ------- ------- |     @staticmethod

0006 0005 0004 0000 0000 0001 0001 04 01 000 000 000 000 000 000 0000.00 0000.00 0000.00 |     def hash_digest(path: Path) -> str:

0006 0005 0004 0000 0000 0001 0001 04 01 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         """Return hash digest for path."""

0006 0005 0004 0000 0000 0001 0001 04 01 000 000 000 000 000 000 0000.00 0000.00 0000.00 | 

0006 0005 0004 0000 0000 0001 0001 04 01 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         data = path.read_bytes()

0006 0005 0004 0000 0000 0001 0001 04 01 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         return hashlib.sha256(data).hexdigest()

0006 0005 0004 0000 0000 0001 0001 04 -- --- --- --- --- --- --- ------- ------- ------- | 

---- ---- ---- ---- ---- ---- ---- 04 -- --- --- --- --- --- --- ------- ------- ------- |     @staticmethod

0007 0006 0005 0000 0000 0001 0001 04 01 000 000 000 000 000 000 0000.00 0000.00 0000.00 |     def get_file_data(path: Path) -> FileData:

0007 0006 0005 0000 0000 0001 0001 04 01 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         """Return file data for path."""

0007 0006 0005 0000 0000 0001 0001 04 01 000 000 000 000 000 000 0000.00 0000.00 0000.00 | 

0007 0006 0005 0000 0000 0001 0001 04 01 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         stat = path.stat()

0007 0006 0005 0000 0000 0001 0001 04 01 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         hash = Cache.hash_digest(path)

0007 0006 0005 0000 0000 0001 0001 04 01 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         return FileData(stat.st_mtime, stat.st_size, hash)

0007 0006 0005 0000 0000 0001 0001 04 -- --- --- --- --- --- --- ------- ------- ------- | 

0015 0014 0013 0000 0000 0001 0001 04 05 002 006 004 008 008 012 0036.00 0048.00 0001.33 |     def is_changed(self, source: Path) -> bool:

0015 0014 0013 0000 0000 0001 0001 04 05 002 006 004 008 008 012 0036.00 0048.00 0001.33 |         """Check if source has changed compared to cached version."""

0015 0014 0013 0000 0000 0001 0001 04 05 002 006 004 008 008 012 0036.00 0048.00 0001.33 |         res_src = source.resolve()

0015 0014 0013 0000 0000 0001 0001 04 05 002 006 004 008 008 012 0036.00 0048.00 0001.33 |         old = self.file_data.get(str(res_src))

0015 0014 0013 0000 0000 0001 0001 04 05 002 006 004 008 008 012 0036.00 0048.00 0001.33 |         if old is None:

0015 0014 0013 0000 0000 0001 0001 04 05 002 006 004 008 008 012 0036.00 0048.00 0001.33 |             return True

0015 0014 0013 0000 0000 0001 0001 04 05 002 006 004 008 008 012 0036.00 0048.00 0001.33 | 

0015 0014 0013 0000 0000 0001 0001 04 05 002 006 004 008 008 012 0036.00 0048.00 0001.33 |         st = res_src.stat()

0015 0014 0013 0000 0000 0001 0001 04 05 002 006 004 008 008 012 0036.00 0048.00 0001.33 |         if st.st_size != old.st_size:

0015 0014 0013 0000 0000 0001 0001 04 05 002 006 004 008 008 012 0036.00 0048.00 0001.33 |             return True

0015 0014 0013 0000 0000 0001 0001 04 05 002 006 004 008 008 012 0036.00 0048.00 0001.33 |         if st.st_mtime != old.st_mtime:

0015 0014 0013 0000 0000 0001 0001 04 05 002 006 004 008 008 012 0036.00 0048.00 0001.33 |             new_hash = Cache.hash_digest(res_src)

0015 0014 0013 0000 0000 0001 0001 04 05 002 006 004 008 008 012 0036.00 0048.00 0001.33 |             if new_hash != old.hash:

0015 0014 0013 0000 0000 0001 0001 04 05 002 006 004 008 008 012 0036.00 0048.00 0001.33 |                 return True

0015 0014 0013 0000 0000 0001 0001 04 05 002 006 004 008 008 012 0036.00 0048.00 0001.33 |         return False

---- ---- ---- ---- ---- ---- ---- 04 -- --- --- --- --- --- --- ------- ------- ------- | 

0014 0012 0009 0000 0004 0001 0000 04 03 000 000 000 000 000 000 0000.00 0000.00 0000.00 |     def filtered_cached(self, sources: Iterable[Path]) -> Tuple[Set[Path], Set[Path]]:

0014 0012 0009 0000 0004 0001 0000 04 03 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         """Split an iterable of paths in `sources` into two sets.

0014 0012 0009 0000 0004 0001 0000 04 03 000 000 000 000 000 000 0000.00 0000.00 0000.00 | 

0014 0012 0009 0000 0004 0001 0000 04 03 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         The first contains paths of files that modified on disk or are not in the

0014 0012 0009 0000 0004 0001 0000 04 03 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         cache. The other contains paths to non-modified files.

0014 0012 0009 0000 0004 0001 0000 04 03 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         """

0014 0012 0009 0000 0004 0001 0000 04 03 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         changed: Set[Path] = set()

0014 0012 0009 0000 0004 0001 0000 04 03 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         done: Set[Path] = set()

0014 0012 0009 0000 0004 0001 0000 04 03 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         for src in sources:

0014 0012 0009 0000 0004 0001 0000 04 03 000 000 000 000 000 000 0000.00 0000.00 0000.00 |             if self.is_changed(src):

0014 0012 0009 0000 0004 0001 0000 04 03 000 000 000 000 000 000 0000.00 0000.00 0000.00 |                 changed.add(src)

0014 0012 0009 0000 0004 0001 0000 04 03 000 000 000 000 000 000 0000.00 0000.00 0000.00 |             else:

0014 0012 0009 0000 0004 0001 0000 04 03 000 000 000 000 000 000 0000.00 0000.00 0000.00 |                 done.add(src)

0014 0012 0009 0000 0004 0001 0000 04 03 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         return changed, done

---- ---- ---- ---- ---- ---- ---- 04 -- --- --- --- --- --- --- ------- ------- ------- | 

0019 0013 0016 0002 0000 0000 0003 04 04 000 000 000 000 000 000 0000.00 0000.00 0000.00 |     def write(self, sources: Iterable[Path]) -> None:

0019 0013 0016 0002 0000 0000 0003 04 04 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         """Update the cache file data and write a new cache file."""

0019 0013 0016 0002 0000 0000 0003 04 04 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         self.file_data.update(

0019 0013 0016 0002 0000 0000 0003 04 04 000 000 000 000 000 000 0000.00 0000.00 0000.00 |             **{str(src.resolve()): Cache.get_file_data(src) for src in sources}

0019 0013 0016 0002 0000 0000 0003 04 04 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         )

0019 0013 0016 0002 0000 0000 0003 04 04 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         try:

0019 0013 0016 0002 0000 0000 0003 04 04 000 000 000 000 000 000 0000.00 0000.00 0000.00 |             CACHE_DIR.mkdir(parents=True, exist_ok=True)

0019 0013 0016 0002 0000 0000 0003 04 04 000 000 000 000 000 000 0000.00 0000.00 0000.00 |             with tempfile.NamedTemporaryFile(

0019 0013 0016 0002 0000 0000 0003 04 04 000 000 000 000 000 000 0000.00 0000.00 0000.00 |                 dir=str(self.cache_file.parent), delete=False

0019 0013 0016 0002 0000 0000 0003 04 04 000 000 000 000 000 000 0000.00 0000.00 0000.00 |             ) as f:

0019 0013 0016 0002 0000 0000 0003 04 04 000 000 000 000 000 000 0000.00 0000.00 0000.00 |                 # We store raw tuples in the cache because pickling NamedTuples

0019 0013 0016 0002 0000 0000 0003 04 04 000 000 000 000 000 000 0000.00 0000.00 0000.00 |                 # doesn't work with mypyc on Python 3.8, and because it's faster.

0019 0013 0016 0002 0000 0000 0003 04 04 000 000 000 000 000 000 0000.00 0000.00 0000.00 |                 data: Dict[str, Tuple[float, int, str]] = {

0019 0013 0016 0002 0000 0000 0003 04 04 000 000 000 000 000 000 0000.00 0000.00 0000.00 |                     k: (*v,) for k, v in self.file_data.items()

0019 0013 0016 0002 0000 0000 0003 04 04 000 000 000 000 000 000 0000.00 0000.00 0000.00 |                 }

0019 0013 0016 0002 0000 0000 0003 04 04 000 000 000 000 000 000 0000.00 0000.00 0000.00 |                 pickle.dump(data, f, protocol=4)

0019 0013 0016 0002 0000 0000 0003 04 04 000 000 000 000 000 000 0000.00 0000.00 0000.00 |             os.replace(f.name, self.cache_file)

0019 0013 0016 0002 0000 0000 0003 04 04 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         except OSError:

0019 0013 0016 0002 0000 0000 0003 04 04 000 000 000 000 000 000 0000.00 0000.00 0000.00 |             pass