wily\state.py
  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
-- -- --- --- --- --- --- --- ------- ------- ------- | """

-- -- --- --- --- --- --- --- ------- ------- ------- | For managing the state of the wily process.

-- -- --- --- --- --- --- --- ------- ------- ------- | 

-- -- --- --- --- --- --- --- ------- ------- ------- | Contains a lazy revision, index and process state model.

-- -- --- --- --- --- --- --- ------- ------- ------- | """

-- -- --- --- --- --- --- --- ------- ------- ------- | from collections import OrderedDict

-- -- --- --- --- --- --- --- ------- ------- ------- | from dataclasses import dataclass, asdict

-- -- --- --- --- --- --- --- ------- ------- ------- | from typing import List

-- -- --- --- --- --- --- --- ------- ------- ------- | 

-- -- --- --- --- --- --- --- ------- ------- ------- | import wily.cache as cache

-- -- --- --- --- --- --- --- ------- ------- ------- | from wily import logger

-- -- --- --- --- --- --- --- ------- ------- ------- | from wily.archivers import Revision, resolve_archiver

-- -- --- --- --- --- --- --- ------- ------- ------- | from wily.operators import get_metric

-- -- --- --- --- --- --- --- ------- ------- ------- | 

-- -- --- --- --- --- --- --- ------- ------- ------- | 

-- -- --- --- --- --- --- --- ------- ------- ------- | @dataclass

02 -- --- --- --- --- --- --- ------- ------- ------- | class IndexedRevision(object):

02 -- --- --- --- --- --- --- ------- ------- ------- |     """Union of revision and the operators executed."""

02 -- --- --- --- --- --- --- ------- ------- ------- | 

02 -- --- --- --- --- --- --- ------- ------- ------- |     revision: Revision

02 -- --- --- --- --- --- --- ------- ------- ------- |     operators: List

02 -- --- --- --- --- --- --- ------- ------- ------- |     _data = None

02 -- --- --- --- --- --- --- ------- ------- ------- | 

02 -- --- --- --- --- --- --- ------- ------- ------- |     @staticmethod

02 02 001 002 001 002 003 003 0004.75 0002.38 0000.50 |     def fromdict(d):

02 02 001 002 001 002 003 003 0004.75 0002.38 0000.50 |         """Instantiate from a dictionary."""

02 02 001 002 001 002 003 003 0004.75 0002.38 0000.50 |         rev = Revision(

02 02 001 002 001 002 003 003 0004.75 0002.38 0000.50 |             key=d["key"],

02 02 001 002 001 002 003 003 0004.75 0002.38 0000.50 |             author_name=d["author_name"],

02 02 001 002 001 002 003 003 0004.75 0002.38 0000.50 |             author_email=d["author_email"],

02 02 001 002 001 002 003 003 0004.75 0002.38 0000.50 |             date=d["date"],

02 02 001 002 001 002 003 003 0004.75 0002.38 0000.50 |             message=d["message"],

02 02 001 002 001 002 003 003 0004.75 0002.38 0000.50 |             files=d["files"] if "files" in d else [],

02 02 001 002 001 002 003 003 0004.75 0002.38 0000.50 |         )

02 02 001 002 001 002 003 003 0004.75 0002.38 0000.50 |         operators = d["operators"]

02 02 001 002 001 002 003 003 0004.75 0002.38 0000.50 |         return IndexedRevision(revision=rev, operators=operators)

02 -- --- --- --- --- --- --- ------- ------- ------- | 

02 01 000 000 000 000 000 000 0000.00 0000.00 0000.00 |     def asdict(self):

02 01 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         """Convert to dictionary."""

02 01 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         d = asdict(self.revision)

02 01 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         d["operators"] = self.operators

02 01 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         return d

02 -- --- --- --- --- --- --- ------- ------- ------- | 

02 02 001 001 001 001 002 002 0002.00 0001.00 0000.50 |     def get(self, config, archiver, operator, path, key):

02 02 001 001 001 001 002 002 0002.00 0001.00 0000.50 |         """

02 02 001 001 001 001 002 002 0002.00 0001.00 0000.50 |         Get the metric data for this indexed revision.

02 02 001 001 001 001 002 002 0002.00 0001.00 0000.50 | 

02 02 001 001 001 001 002 002 0002.00 0001.00 0000.50 |         :param config: The wily config.

02 02 001 001 001 001 002 002 0002.00 0001.00 0000.50 |         :type  config: :class:`wily.config.WilyConfig`

02 02 001 001 001 001 002 002 0002.00 0001.00 0000.50 | 

02 02 001 001 001 001 002 002 0002.00 0001.00 0000.50 |         :param archiver: The archiver.

02 02 001 001 001 001 002 002 0002.00 0001.00 0000.50 |         :type  archiver: :class:`wily.archivers.Archiver`

02 02 001 001 001 001 002 002 0002.00 0001.00 0000.50 | 

02 02 001 001 001 001 002 002 0002.00 0001.00 0000.50 |         :param operator: The operator to find

02 02 001 001 001 001 002 002 0002.00 0001.00 0000.50 |         :type  operator: ``str``

02 02 001 001 001 001 002 002 0002.00 0001.00 0000.50 | 

02 02 001 001 001 001 002 002 0002.00 0001.00 0000.50 |         :param path: The path to find

02 02 001 001 001 001 002 002 0002.00 0001.00 0000.50 |         :type  path: ``str``

02 02 001 001 001 001 002 002 0002.00 0001.00 0000.50 | 

02 02 001 001 001 001 002 002 0002.00 0001.00 0000.50 |         :param key: The metric key

02 02 001 001 001 001 002 002 0002.00 0001.00 0000.50 |         :type  key: ``str``

02 02 001 001 001 001 002 002 0002.00 0001.00 0000.50 |         """

02 02 001 001 001 001 002 002 0002.00 0001.00 0000.50 |         if not self._data:

02 02 001 001 001 001 002 002 0002.00 0001.00 0000.50 |             self._data = cache.get(

02 02 001 001 001 001 002 002 0002.00 0001.00 0000.50 |                 config=config, archiver=archiver, revision=self.revision.key

02 02 001 001 001 001 002 002 0002.00 0001.00 0000.50 |             )["operator_data"]

02 02 001 001 001 001 002 002 0002.00 0001.00 0000.50 |         logger.debug(f"Fetching metric {path} - {key} for operator {operator}")

02 02 001 001 001 001 002 002 0002.00 0001.00 0000.50 |         return get_metric(self._data, operator, path, key)

02 -- --- --- --- --- --- --- ------- ------- ------- | 

02 01 000 000 000 000 000 000 0000.00 0000.00 0000.00 |     def store(self, config, archiver, stats):

02 01 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         """

02 01 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         Store the stats for this indexed revision.

02 01 000 000 000 000 000 000 0000.00 0000.00 0000.00 | 

02 01 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         :param config: The wily config.

02 01 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         :type  config: :class:`wily.config.WilyConfig`

02 01 000 000 000 000 000 000 0000.00 0000.00 0000.00 | 

02 01 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         :param archiver: The archiver.

02 01 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         :type  archiver: :class:`wily.archivers.Archiver`

02 01 000 000 000 000 000 000 0000.00 0000.00 0000.00 | 

02 01 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         :param stats: The data

02 01 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         :type  stats: ``dict``

02 01 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         """

02 01 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         self._data = stats

02 01 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         return cache.store(config, archiver, self.revision, stats)

-- -- --- --- --- --- --- --- ------- ------- ------- | 

-- -- --- --- --- --- --- --- ------- ------- ------- | 

02 -- --- --- --- --- --- --- ------- ------- ------- | class Index(object):

02 -- --- --- --- --- --- --- ------- ------- ------- |     """The index of the wily cache."""

02 -- --- --- --- --- --- --- ------- ------- ------- | 

02 -- --- --- --- --- --- --- ------- ------- ------- |     operators = None

02 -- --- --- --- --- --- --- ------- ------- ------- | 

02 03 000 000 000 000 000 000 0000.00 0000.00 0000.00 |     def __init__(self, config, archiver):

02 03 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         """

02 03 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         Instantiate a new index.

02 03 000 000 000 000 000 000 0000.00 0000.00 0000.00 | 

02 03 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         :param config: The wily config.

02 03 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         :type  config: :class:`wily.config.WilyConfig`

02 03 000 000 000 000 000 000 0000.00 0000.00 0000.00 | 

02 03 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         :param archiver: The archiver.

02 03 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         :type  archiver: :class:`wily.archivers.Archiver`

02 03 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         """

02 03 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         self.config = config

02 03 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         self.archiver = archiver

02 03 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         self.data = (

02 03 000 000 000 000 000 000 0000.00 0000.00 0000.00 |             cache.get_archiver_index(config, archiver.name)

02 03 000 000 000 000 000 000 0000.00 0000.00 0000.00 |             if cache.has_archiver_index(config, archiver.name)

02 03 000 000 000 000 000 000 0000.00 0000.00 0000.00 |             else []

02 03 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         )

02 03 000 000 000 000 000 000 0000.00 0000.00 0000.00 | 

02 03 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         self._revisions = OrderedDict(

02 03 000 000 000 000 000 000 0000.00 0000.00 0000.00 |             {d["key"]: IndexedRevision.fromdict(d) for d in self.data}

02 -- 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         )

02 -- --- --- --- --- --- --- ------- ------- ------- | 

02 01 000 000 000 000 000 000 0000.00 0000.00 0000.00 |     def __len__(self):

02 01 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         """Use length of revisions as len."""

02 01 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         return len(self._revisions)

02 -- --- --- --- --- --- --- ------- ------- ------- | 

02 -- --- --- --- --- --- --- ------- ------- ------- |     @property

02 01 000 000 000 000 000 000 0000.00 0000.00 0000.00 |     def last_revision(self):

02 01 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         """

02 01 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         Return the most recent revision.

02 01 000 000 000 000 000 000 0000.00 0000.00 0000.00 | 

02 01 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         :rtype: ``list`` of :class:`LazyRevision`

02 01 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         """

02 01 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         return next(iter(self._revisions.values()))

02 -- --- --- --- --- --- --- ------- ------- ------- | 

02 -- --- --- --- --- --- --- ------- ------- ------- |     @property

02 01 000 000 000 000 000 000 0000.00 0000.00 0000.00 |     def revisions(self):

02 01 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         """

02 01 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         List of all the revisions.

02 01 000 000 000 000 000 000 0000.00 0000.00 0000.00 | 

02 01 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         :rtype: ``list`` of :class:`LazyRevision`

02 01 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         """

02 01 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         return list(self._revisions.values())

02 -- --- --- --- --- --- --- ------- ------- ------- | 

02 -- --- --- --- --- --- --- ------- ------- ------- |     @property

02 01 000 000 000 000 000 000 0000.00 0000.00 0000.00 |     def revision_keys(self):

02 01 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         """

02 01 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         List of all the revision indexes.

02 01 000 000 000 000 000 000 0000.00 0000.00 0000.00 | 

02 01 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         :rtype: ``list`` of ``str``

02 01 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         """

02 01 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         return list(self._revisions.keys())

02 -- --- --- --- --- --- --- ------- ------- ------- | 

02 03 001 003 002 004 004 006 0012.00 0008.00 0000.67 |     def __contains__(self, item):

02 03 001 003 002 004 004 006 0012.00 0008.00 0000.67 |         """

02 03 001 003 002 004 004 006 0012.00 0008.00 0000.67 |         Check if index contains `item`.

02 03 001 003 002 004 004 006 0012.00 0008.00 0000.67 | 

02 03 001 003 002 004 004 006 0012.00 0008.00 0000.67 |         :param item: The item to search for

02 03 001 003 002 004 004 006 0012.00 0008.00 0000.67 |         :type  item: ``str``, :class:`Revision` or :class:`LazyRevision`

02 03 001 003 002 004 004 006 0012.00 0008.00 0000.67 | 

02 03 001 003 002 004 004 006 0012.00 0008.00 0000.67 |         :return: ``True`` for contains, ``False`` for not.

02 03 001 003 002 004 004 006 0012.00 0008.00 0000.67 |         """

02 03 001 003 002 004 004 006 0012.00 0008.00 0000.67 |         if isinstance(item, Revision):

02 03 001 003 002 004 004 006 0012.00 0008.00 0000.67 |             return item.key in self._revisions

02 03 001 003 002 004 004 006 0012.00 0008.00 0000.67 |         elif isinstance(item, str):

02 03 001 003 002 004 004 006 0012.00 0008.00 0000.67 |             return item in self._revisions

02 03 001 003 002 004 004 006 0012.00 0008.00 0000.67 |         else:

02 03 001 003 002 004 004 006 0012.00 0008.00 0000.67 |             raise TypeError("Invalid type for __contains__ in Index.")

02 -- --- --- --- --- --- --- ------- ------- ------- | 

02 01 000 000 000 000 000 000 0000.00 0000.00 0000.00 |     def __getitem__(self, index):

02 01 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         """Get the revision for a specific index."""

02 01 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         return self._revisions[index]

02 -- --- --- --- --- --- --- ------- ------- ------- | 

02 02 000 000 000 000 000 000 0000.00 0000.00 0000.00 |     def add(self, revision, operators):

02 02 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         """

02 02 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         Add a revision to the index.

02 02 000 000 000 000 000 000 0000.00 0000.00 0000.00 | 

02 02 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         :param revision: The revision.

02 02 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         :type  revision: :class:`Revision` or :class:`LazyRevision`

02 02 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         """

02 02 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         ir = IndexedRevision(

02 02 000 000 000 000 000 000 0000.00 0000.00 0000.00 |             revision=revision, operators=[operator.name for operator in operators]

02 02 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         )

02 02 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         self._revisions[revision.key] = ir

02 02 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         return ir

02 -- --- --- --- --- --- --- ------- ------- ------- | 

02 02 000 000 000 000 000 000 0000.00 0000.00 0000.00 |     def save(self):

02 02 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         """Save the index data back to the wily cache."""

02 02 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         data = [i.asdict() for i in self._revisions.values()]

02 02 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         logger.debug("Saving data")

02 02 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         cache.store_archiver_index(self.config, self.archiver, data)

-- -- --- --- --- --- --- --- ------- ------- ------- | 

-- -- --- --- --- --- --- --- ------- ------- ------- | 

04 -- --- --- --- --- --- --- ------- ------- ------- | class State(object):

04 -- --- --- --- --- --- --- ------- ------- ------- |     """

04 -- --- --- --- --- --- --- ------- ------- ------- |     The wily process state.

04 -- --- --- --- --- --- --- ------- ------- ------- | 

04 -- --- --- --- --- --- --- ------- ------- ------- |     Includes indexes for each archiver.

04 -- --- --- --- --- --- --- ------- ------- ------- |     """

04 -- --- --- --- --- --- --- ------- ------- ------- | 

04 03 000 000 000 000 000 000 0000.00 0000.00 0000.00 |     def __init__(self, config, archiver=None):

04 03 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         """

04 03 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         Instantiate a new process state.

04 03 000 000 000 000 000 000 0000.00 0000.00 0000.00 | 

04 03 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         :param config: The wily configuration.

04 03 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         :type  config: :class:`WilyConfig`

04 03 000 000 000 000 000 000 0000.00 0000.00 0000.00 | 

04 03 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         :param archiver: The archiver (optional).

04 03 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         :type  archiver: :class:`wily.archivers.Archiver`

04 03 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         """

04 03 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         if archiver:

04 03 000 000 000 000 000 000 0000.00 0000.00 0000.00 |             self.archivers = [archiver.name]

04 03 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         else:

04 03 000 000 000 000 000 000 0000.00 0000.00 0000.00 |             self.archivers = cache.list_archivers(config)

04 03 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         logger.debug(f"Initialised state indexes for archivers {self.archivers}")

04 03 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         self.config = config

04 03 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         self.index = {}

04 03 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         for archiver in self.archivers:

04 03 000 000 000 000 000 000 0000.00 0000.00 0000.00 |             self.index[archiver] = Index(self.config, resolve_archiver(archiver))

04 03 000 000 000 000 000 000 0000.00 0000.00 0000.00 |         self.default_archiver = self.archivers[0]

04 -- --- --- --- --- --- --- ------- ------- ------- | 

04 02 001 001 001 001 002 002 0002.00 0001.00 0000.50 |     def ensure_exists(self):

04 02 001 001 001 001 002 002 0002.00 0001.00 0000.50 |         """Ensure that cache directory exists."""

04 02 001 001 001 001 002 002 0002.00 0001.00 0000.50 |         if not cache.exists(self.config):

04 02 001 001 001 001 002 002 0002.00 0001.00 0000.50 |             logger.debug("Wily cache not found, creating.")

04 02 001 001 001 001 002 002 0002.00 0001.00 0000.50 |             cache.create(self.config)

04 02 001 001 001 001 002 002 0002.00 0001.00 0000.50 |             logger.debug("Created wily cache")

04 02 001 001 001 001 002 002 0002.00 0001.00 0000.50 |         else:

04 02 001 001 001 001 002 002 0002.00 0001.00 0000.50 |             logger.debug(f"Cache {self.config.cache_path} exists")