src\wily\state.py
  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
-- -- --- --- --- --- --- --- ------- ------- ------- | """

-- -- --- --- --- --- --- --- ------- ------- ------- | For managing the state of the wily process.

-- -- --- --- --- --- --- --- ------- ------- ------- | 

-- -- --- --- --- --- --- --- ------- ------- ------- | Contains a lazy revision, index and process state model.

-- -- --- --- --- --- --- --- ------- ------- ------- | """

-- -- --- --- --- --- --- --- ------- ------- ------- | from collections import OrderedDict

-- -- --- --- --- --- --- --- ------- ------- ------- | from dataclasses import asdict, dataclass

-- -- --- --- --- --- --- --- ------- ------- ------- | from pathlib import Path

-- -- --- --- --- --- --- --- ------- ------- ------- | from typing import Any, Dict, List, Optional, Union

-- -- --- --- --- --- --- --- ------- ------- ------- | 

-- -- --- --- --- --- --- --- ------- ------- ------- | from wily import cache, logger

-- -- --- --- --- --- --- --- ------- ------- ------- | from wily.archivers import Archiver, BaseArchiver, Revision, resolve_archiver

-- -- --- --- --- --- --- --- ------- ------- ------- | from wily.config.types import WilyConfig

-- -- --- --- --- --- --- --- ------- ------- ------- | from wily.operators import Operator, get_metric

-- -- --- --- --- --- --- --- ------- ------- ------- | 

-- -- --- --- --- --- --- --- ------- ------- ------- | 

-- -- --- --- --- --- --- --- ------- ------- ------- | @dataclass

03 -- --- --- --- --- --- --- ------- ------- ------- | class IndexedRevision:

03 -- --- --- --- --- --- --- ------- ------- ------- |     """Union of revision and the operators executed."""

03 -- --- --- --- --- --- --- ------- ------- ------- | 

03 -- --- --- --- --- --- --- ------- ------- ------- |     revision: Revision

03 -- --- --- --- --- --- --- ------- ------- ------- |     operators: List

03 -- --- --- --- --- --- --- ------- ------- ------- |     _data = None

03 -- --- --- --- --- --- --- ------- ------- ------- | 

03 -- --- --- --- --- --- --- ------- ------- ------- |     @staticmethod

03 06 001 006 005 010 007 015 0042.11 0035.09 0000.83 |     def fromdict(d: Dict[str, Any]) -> "IndexedRevision":

03 06 001 006 005 010 007 015 0042.11 0035.09 0000.83 |         """Instantiate from a dictionary."""

03 06 001 006 005 010 007 015 0042.11 0035.09 0000.83 |         rev = Revision(

03 06 001 006 005 010 007 015 0042.11 0035.09 0000.83 |             key=d["key"],

03 06 001 006 005 010 007 015 0042.11 0035.09 0000.83 |             author_name=d["author_name"],

03 06 001 006 005 010 007 015 0042.11 0035.09 0000.83 |             author_email=d["author_email"],

03 06 001 006 005 010 007 015 0042.11 0035.09 0000.83 |             date=d["date"],

03 06 001 006 005 010 007 015 0042.11 0035.09 0000.83 |             message=d["message"],

03 06 001 006 005 010 007 015 0042.11 0035.09 0000.83 |             tracked_files=d["tracked_files"] if "tracked_files" in d else [],

03 06 001 006 005 010 007 015 0042.11 0035.09 0000.83 |             tracked_dirs=d["tracked_dirs"] if "tracked_dirs" in d else [],

03 06 001 006 005 010 007 015 0042.11 0035.09 0000.83 |             added_files=d["added_files"] if "added_files" in d else [],

03 06 001 006 005 010 007 015 0042.11 0035.09 0000.83 |             modified_files=d["modified_files"] if "modified_files" in d else [],

03 06 001 006 005 010 007 015 0042.11 0035.09 0000.83 |             deleted_files=d["deleted_files"] if "deleted_files" in d else [],

03 06 001 006 005 010 007 015 0042.11 0035.09 0000.83 |         )

03 06 001 006 005 010 007 015 0042.11 0035.09 0000.83 |         operators = d["operators"]

03 06 001 006 005 010 007 015 0042.11 0035.09 0000.83 |         return IndexedRevision(revision=rev, operators=operators)

03 -- --- --- --- --- --- --- ------- ------- ------- | 

03 01 000 000 000 000 000 000 0000.00 0000.00 0000.00 |     def asdict(self) -> Dict[str, Any]:

03 01 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         """Convert to dictionary."""

03 01 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         d = asdict(self.revision)

03 01 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         d["operators"] = self.operators

03 01 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         return d

03 -- --- --- --- --- --- --- ------- ------- ------- | 

03 02 001 001 001 001 002 002 0002.00 0001.00 0000.50 |     def get(

03 02 001 001 001 001 002 002 0002.00 0001.00 0000.50 |         self, config: WilyConfig, archiver: str, operator: str, path: str, key: str

03 02 001 001 001 001 002 002 0002.00 0001.00 0000.50 |     ) -> Any:

03 02 001 001 001 001 002 002 0002.00 0001.00 0000.50 |         """

03 02 001 001 001 001 002 002 0002.00 0001.00 0000.50 |         Get the metric data for this indexed revision.

03 02 001 001 001 001 002 002 0002.00 0001.00 0000.50 | 

03 02 001 001 001 001 002 002 0002.00 0001.00 0000.50 |         :param config: The wily config.

03 02 001 001 001 001 002 002 0002.00 0001.00 0000.50 |         :param archiver: The archiver.

03 02 001 001 001 001 002 002 0002.00 0001.00 0000.50 |         :param operator: The operator to find

03 02 001 001 001 001 002 002 0002.00 0001.00 0000.50 |         :param path: The path to find

03 02 001 001 001 001 002 002 0002.00 0001.00 0000.50 |         :param key: The metric key

03 02 001 001 001 001 002 002 0002.00 0001.00 0000.50 |         """

03 02 001 001 001 001 002 002 0002.00 0001.00 0000.50 |         if not self._data:

03 02 001 001 001 001 002 002 0002.00 0001.00 0000.50 |             self._data = cache.get(

03 02 001 001 001 001 002 002 0002.00 0001.00 0000.50 |                 config=config, archiver=archiver, revision=self.revision.key

03 02 001 001 001 001 002 002 0002.00 0001.00 0000.50 |             )["operator_data"]

03 02 001 001 001 001 002 002 0002.00 0001.00 0000.50 |         logger.debug(f"Fetching metric {path} - {key} for operator {operator}")

03 02 001 001 001 001 002 002 0002.00 0001.00 0000.50 |         return get_metric(self._data, operator, path, key)

03 -- --- --- --- --- --- --- ------- ------- ------- | 

03 02 001 001 001 001 002 002 0002.00 0001.00 0000.50 |     def get_paths(self, config: WilyConfig, archiver: str, operator: str) -> List[str]:

03 02 001 001 001 001 002 002 0002.00 0001.00 0000.50 |         """

03 02 001 001 001 001 002 002 0002.00 0001.00 0000.50 |         Get the indexed paths for this indexed revision.

03 02 001 001 001 001 002 002 0002.00 0001.00 0000.50 | 

03 02 001 001 001 001 002 002 0002.00 0001.00 0000.50 |         :param config: The wily config.

03 02 001 001 001 001 002 002 0002.00 0001.00 0000.50 |         :param archiver: The archiver.

03 02 001 001 001 001 002 002 0002.00 0001.00 0000.50 |         :param operator: The operator to find

03 02 001 001 001 001 002 002 0002.00 0001.00 0000.50 | 

03 02 001 001 001 001 002 002 0002.00 0001.00 0000.50 |         :return: A list of paths

03 02 001 001 001 001 002 002 0002.00 0001.00 0000.50 |         """

03 02 001 001 001 001 002 002 0002.00 0001.00 0000.50 |         if not self._data:

03 02 001 001 001 001 002 002 0002.00 0001.00 0000.50 |             self._data = cache.get(

03 02 001 001 001 001 002 002 0002.00 0001.00 0000.50 |                 config=config, archiver=archiver, revision=self.revision.key

03 02 001 001 001 001 002 002 0002.00 0001.00 0000.50 |             )["operator_data"]

03 02 001 001 001 001 002 002 0002.00 0001.00 0000.50 |         logger.debug("Fetching keys")

03 02 001 001 001 001 002 002 0002.00 0001.00 0000.50 |         return list(self._data[operator].keys())

03 -- --- --- --- --- --- --- ------- ------- ------- | 

03 01 000 000 000 000 000 000 0000.00 0000.00 0000.00 |     def store(

03 01 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         self, config: WilyConfig, archiver: Union[Archiver, str], stats: Dict[str, Any]

03 01 000 000 000 000 000 000 0000.00 0000.00 0000.00 |     ) -> Path:

03 01 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         """

03 01 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         Store the stats for this indexed revision.

03 01 000 000 000 000 000 000 0000.00 0000.00 0000.00 | 

03 01 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         :param config: The wily config.

03 01 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         :param archiver: The archiver.

03 01 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         :param stats: The data

03 01 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         """

03 01 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         self._data = stats

03 01 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         return cache.store(config, archiver, self.revision, stats)

-- -- --- --- --- --- --- --- ------- ------- ------- | 

-- -- --- --- --- --- --- --- ------- ------- ------- | 

02 -- --- --- --- --- --- --- ------- ------- ------- | class Index:

02 -- --- --- --- --- --- --- ------- ------- ------- |     """The index of the wily cache."""

02 -- --- --- --- --- --- --- ------- ------- ------- | 

02 -- --- --- --- --- --- --- ------- ------- ------- |     archiver: Archiver

02 -- --- --- --- --- --- --- ------- ------- ------- |     config: WilyConfig

02 -- --- --- --- --- --- --- ------- ------- ------- |     operators = None

02 -- --- --- --- --- --- --- ------- ------- ------- |     data: List[Any]

02 -- --- --- --- --- --- --- ------- ------- ------- | 

02 03 000 000 000 000 000 000 0000.00 0000.00 0000.00 |     def __init__(self, config: WilyConfig, archiver: Archiver):

02 03 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         """

02 03 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         Instantiate a new index.

02 03 000 000 000 000 000 000 0000.00 0000.00 0000.00 | 

02 03 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         :param config: The wily config.

02 03 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         :param archiver: The archiver.

02 03 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         """

02 03 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         self.config = config

02 03 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         self.archiver = archiver

02 03 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         self.data = (

02 03 000 000 000 000 000 000 0000.00 0000.00 0000.00 |             cache.get_archiver_index(config, archiver.name)

02 03 000 000 000 000 000 000 0000.00 0000.00 0000.00 |             if cache.has_archiver_index(config, archiver.name)

02 03 000 000 000 000 000 000 0000.00 0000.00 0000.00 |             else []

02 03 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         )

02 03 000 000 000 000 000 000 0000.00 0000.00 0000.00 | 

02 03 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         self._revisions = OrderedDict(

02 03 000 000 000 000 000 000 0000.00 0000.00 0000.00 |             {d["key"]: IndexedRevision.fromdict(d) for d in self.data}

02 -- 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         )

02 -- --- --- --- --- --- --- ------- ------- ------- | 

02 01 000 000 000 000 000 000 0000.00 0000.00 0000.00 |     def __len__(self):

02 01 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         """Use length of revisions as len."""

02 01 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         return len(self._revisions)

02 -- --- --- --- --- --- --- ------- ------- ------- | 

02 -- --- --- --- --- --- --- ------- ------- ------- |     @property

02 01 000 000 000 000 000 000 0000.00 0000.00 0000.00 |     def last_revision(self) -> IndexedRevision:

02 01 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         """Return the most recent revision."""

02 01 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         return next(iter(self._revisions.values()))

02 -- --- --- --- --- --- --- ------- ------- ------- | 

02 -- --- --- --- --- --- --- ------- ------- ------- |     @property

02 01 000 000 000 000 000 000 0000.00 0000.00 0000.00 |     def revisions(self) -> List[IndexedRevision]:

02 01 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         """List of all the revisions."""

02 01 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         return list(self._revisions.values())

02 -- --- --- --- --- --- --- ------- ------- ------- | 

02 -- --- --- --- --- --- --- ------- ------- ------- |     @property

02 01 000 000 000 000 000 000 0000.00 0000.00 0000.00 |     def revision_keys(self) -> List[str]:

02 01 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         """List of all the revision indexes."""

02 01 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         return list(self._revisions.keys())

02 -- --- --- --- --- --- --- ------- ------- ------- | 

02 03 001 003 002 004 004 006 0012.00 0008.00 0000.67 |     def __contains__(self, item: Union[str, Revision]) -> bool:

02 03 001 003 002 004 004 006 0012.00 0008.00 0000.67 |         """Check if index contains `item`."""

02 03 001 003 002 004 004 006 0012.00 0008.00 0000.67 |         if isinstance(item, Revision):

02 03 001 003 002 004 004 006 0012.00 0008.00 0000.67 |             return item.key in self._revisions

02 03 001 003 002 004 004 006 0012.00 0008.00 0000.67 |         elif isinstance(item, str):

02 03 001 003 002 004 004 006 0012.00 0008.00 0000.67 |             return item in self._revisions

02 03 001 003 002 004 004 006 0012.00 0008.00 0000.67 |         else:

02 03 001 003 002 004 004 006 0012.00 0008.00 0000.67 |             raise TypeError("Invalid type for __contains__ in Index.")

02 -- --- --- --- --- --- --- ------- ------- ------- | 

02 01 000 000 000 000 000 000 0000.00 0000.00 0000.00 |     def __getitem__(self, index) -> IndexedRevision:

02 01 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         """Get the revision for a specific index."""

02 01 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         return self._revisions[index]

02 -- --- --- --- --- --- --- ------- ------- ------- | 

02 02 000 000 000 000 000 000 0000.00 0000.00 0000.00 |     def add(self, revision: Revision, operators: List[Operator]) -> IndexedRevision:

02 02 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         """

02 02 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         Add a revision to the index.

02 02 000 000 000 000 000 000 0000.00 0000.00 0000.00 | 

02 02 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         :param revision: The revision.

02 02 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         :param operators: Operators for the revision.

02 02 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         """

02 02 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         ir = IndexedRevision(

02 02 000 000 000 000 000 000 0000.00 0000.00 0000.00 |             revision=revision, operators=[operator.name for operator in operators]

02 02 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         )

02 02 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         self._revisions[revision.key] = ir

02 02 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         return ir

02 -- --- --- --- --- --- --- ------- ------- ------- | 

02 02 000 000 000 000 000 000 0000.00 0000.00 0000.00 |     def save(self):

02 02 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         """Save the index data back to the wily cache."""

02 02 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         data = [i.asdict() for i in self._revisions.values()]

02 02 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         logger.debug("Saving data")

02 02 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         cache.store_archiver_index(self.config, self.archiver, data)

-- -- --- --- --- --- --- --- ------- ------- ------- | 

-- -- --- --- --- --- --- --- ------- ------- ------- | 

04 -- --- --- --- --- --- --- ------- ------- ------- | class State:

04 -- --- --- --- --- --- --- ------- ------- ------- |     """

04 -- --- --- --- --- --- --- ------- ------- ------- |     The wily process state.

04 -- --- --- --- --- --- --- ------- ------- ------- | 

04 -- --- --- --- --- --- --- ------- ------- ------- |     Includes indexes for each archiver.

04 -- --- --- --- --- --- --- ------- ------- ------- |     """

04 -- --- --- --- --- --- --- ------- ------- ------- | 

04 -- --- --- --- --- --- --- ------- ------- ------- |     archivers: List[str]

04 -- --- --- --- --- --- --- ------- ------- ------- |     config: WilyConfig

04 -- --- --- --- --- --- --- ------- ------- ------- |     index: Dict[str, Index]

04 -- --- --- --- --- --- --- ------- ------- ------- |     default_archiver: str

04 -- --- --- --- --- --- --- ------- ------- ------- |     operators: Optional[List[Operator]] = None

04 -- --- --- --- --- --- --- ------- ------- ------- | 

04 03 000 000 000 000 000 000 0000.00 0000.00 0000.00 |     def __init__(

04 03 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         self,

04 03 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         config: WilyConfig,

04 03 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         archiver: Optional[Union[Archiver, BaseArchiver]] = None,

04 03 000 000 000 000 000 000 0000.00 0000.00 0000.00 |     ):

04 03 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         """

04 03 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         Instantiate a new process state.

04 03 000 000 000 000 000 000 0000.00 0000.00 0000.00 | 

04 03 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         :param config: The wily configuration.

04 03 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         :param archiver: The archiver (optional).

04 03 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         """

04 03 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         if archiver:

04 03 000 000 000 000 000 000 0000.00 0000.00 0000.00 |             self.archivers = [archiver.name]

04 03 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         else:

04 03 000 000 000 000 000 000 0000.00 0000.00 0000.00 |             self.archivers = cache.list_archivers(config)

04 03 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         logger.debug(f"Initialised state indexes for archivers {self.archivers}")

04 03 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         self.config = config

04 03 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         self.index = {}

04 03 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         for _archiver in self.archivers:

04 03 000 000 000 000 000 000 0000.00 0000.00 0000.00 |             self.index[_archiver] = Index(self.config, resolve_archiver(_archiver))

04 03 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         self.default_archiver = self.archivers[0]

04 -- --- --- --- --- --- --- ------- ------- ------- | 

04 02 001 001 001 001 002 002 0002.00 0001.00 0000.50 |     def ensure_exists(self):

04 02 001 001 001 001 002 002 0002.00 0001.00 0000.50 |         """Ensure that cache directory exists."""

04 02 001 001 001 001 002 002 0002.00 0001.00 0000.50 |         if not cache.exists(self.config):

04 02 001 001 001 001 002 002 0002.00 0001.00 0000.50 |             logger.debug("Wily cache not found, creating.")

04 02 001 001 001 001 002 002 0002.00 0001.00 0000.50 |             cache.create(self.config)

04 02 001 001 001 001 002 002 0002.00 0001.00 0000.50 |             logger.debug("Created wily cache")

04 02 001 001 001 001 002 002 0002.00 0001.00 0000.50 |         else:

04 02 001 001 001 001 002 002 0002.00 0001.00 0000.50 |             logger.debug(f"Cache {self.config.cache_path} exists")