1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38 """
39 Implements the standard 'store' action.
40 @sort: executeStore, writeImage, writeStoreIndicator, consistencyCheck
41 @author: Kenneth J. Pronovici <pronovic@ieee.org>
42 @author: Dmitry Rutsky <rutsky@inbox.ru>
43 """
44
45
46
47
48
49
50
51 import sys
52 import os
53 import logging
54 import datetime
55 import tempfile
56
57
58 from CedarBackup3.filesystem import compareContents
59 from CedarBackup3.util import isStartOfWeek
60 from CedarBackup3.util import mount, unmount, displayBytes
61 from CedarBackup3.actions.util import createWriter, checkMediaState, buildMediaLabel, writeIndicatorFile
62 from CedarBackup3.actions.constants import DIR_TIME_FORMAT, STAGE_INDICATOR, STORE_INDICATOR
63
64
65
66
67
68
69 logger = logging.getLogger("CedarBackup3.log.actions.store")
70
71
72
73
74
75
76
77
78
79
81 """
82 Executes the store backup action.
83
84 @note: The rebuild action and the store action are very similar. The
85 main difference is that while store only stores a single day's staging
86 directory, the rebuild action operates on multiple staging directories.
87
88 @note: When the store action is complete, we will write a store indicator to
89 the daily staging directory we used, so it's obvious that the store action
90 has completed.
91
92 @param configPath: Path to configuration file on disk.
93 @type configPath: String representing a path on disk.
94
95 @param options: Program command-line options.
96 @type options: Options object.
97
98 @param config: Program configuration.
99 @type config: Config object.
100
101 @raise ValueError: Under many generic error conditions
102 @raise IOError: If there are problems reading or writing files.
103 """
104 logger.debug("Executing the 'store' action.")
105 if sys.platform == "darwin":
106 logger.warning("Warning: the store action is not fully supported on Mac OS X.")
107 logger.warning("See the Cedar Backup software manual for further information.")
108 if config.options is None or config.store is None:
109 raise ValueError("Store configuration is not properly filled in.")
110 if config.store.checkMedia:
111 checkMediaState(config.store)
112 rebuildMedia = options.full
113 logger.debug("Rebuild media flag [%s]", rebuildMedia)
114 todayIsStart = isStartOfWeek(config.options.startingDay)
115 stagingDirs = _findCorrectDailyDir(options, config)
116 writeImageBlankSafe(config, rebuildMedia, todayIsStart, config.store.blankBehavior, stagingDirs)
117 if config.store.checkData:
118 if sys.platform == "darwin":
119 logger.warning("Warning: consistency check cannot be run successfully on Mac OS X.")
120 logger.warning("See the Cedar Backup software manual for further information.")
121 else:
122 logger.debug("Running consistency check of media.")
123 consistencyCheck(config, stagingDirs)
124 writeStoreIndicator(config, stagingDirs)
125 logger.info("Executed the 'store' action successfully.")
126
127
128
129
130
131
133 """
134 Builds and writes an ISO image containing the indicated stage directories.
135
136 The generated image will contain each of the staging directories listed in
137 C{stagingDirs}. The directories will be placed into the image at the root by
138 date, so staging directory C{/opt/stage/2005/02/10} will be placed into the
139 disc at C{/2005/02/10}.
140
141 @note: This function is implemented in terms of L{writeImageBlankSafe}. The
142 C{newDisc} flag is passed in for both C{rebuildMedia} and C{todayIsStart}.
143
144 @param config: Config object.
145 @param newDisc: Indicates whether the disc should be re-initialized
146 @param stagingDirs: Dictionary mapping directory path to date suffix.
147
148 @raise ValueError: Under many generic error conditions
149 @raise IOError: If there is a problem writing the image to disc.
150 """
151 writeImageBlankSafe(config, newDisc, newDisc, None, stagingDirs)
152
153
154
155
156
157
159 """
160 Builds and writes an ISO image containing the indicated stage directories.
161
162 The generated image will contain each of the staging directories listed in
163 C{stagingDirs}. The directories will be placed into the image at the root by
164 date, so staging directory C{/opt/stage/2005/02/10} will be placed into the
165 disc at C{/2005/02/10}. The media will always be written with a media
166 label specific to Cedar Backup.
167
168 This function is similar to L{writeImage}, but tries to implement a smarter
169 blanking strategy.
170
171 First, the media is always blanked if the C{rebuildMedia} flag is true.
172 Then, if C{rebuildMedia} is false, blanking behavior and C{todayIsStart}
173 come into effect::
174
175 If no blanking behavior is specified, and it is the start of the week,
176 the disc will be blanked
177
178 If blanking behavior is specified, and either the blank mode is "daily"
179 or the blank mode is "weekly" and it is the start of the week, then
180 the disc will be blanked if it looks like the weekly backup will not
181 fit onto the media.
182
183 Otherwise, the disc will not be blanked
184
185 How do we decide whether the weekly backup will fit onto the media? That is
186 what the blanking factor is used for. The following formula is used::
187
188 will backup fit? = (bytes available / (1 + bytes required) <= blankFactor
189
190 The blanking factor will vary from setup to setup, and will probably
191 require some experimentation to get it right.
192
193 @param config: Config object.
194 @param rebuildMedia: Indicates whether media should be rebuilt
195 @param todayIsStart: Indicates whether today is the starting day of the week
196 @param blankBehavior: Blank behavior from configuration, or C{None} to use default behavior
197 @param stagingDirs: Dictionary mapping directory path to date suffix.
198
199 @raise ValueError: Under many generic error conditions
200 @raise IOError: If there is a problem writing the image to disc.
201 """
202 mediaLabel = buildMediaLabel()
203 writer = createWriter(config)
204 writer.initializeImage(True, config.options.workingDir, mediaLabel)
205 for stageDir in list(stagingDirs.keys()):
206 logger.debug("Adding stage directory [%s].", stageDir)
207 dateSuffix = stagingDirs[stageDir]
208 writer.addImageEntry(stageDir, dateSuffix)
209 newDisc = _getNewDisc(writer, rebuildMedia, todayIsStart, blankBehavior)
210 writer.setImageNewDisc(newDisc)
211 writer.writeImage()
212
213 -def _getNewDisc(writer, rebuildMedia, todayIsStart, blankBehavior):
214 """
215 Gets a value for the newDisc flag based on blanking factor rules.
216
217 The blanking factor rules are described above by L{writeImageBlankSafe}.
218
219 @param writer: Previously configured image writer containing image entries
220 @param rebuildMedia: Indicates whether media should be rebuilt
221 @param todayIsStart: Indicates whether today is the starting day of the week
222 @param blankBehavior: Blank behavior from configuration, or C{None} to use default behavior
223
224 @return: newDisc flag to be set on writer.
225 """
226 newDisc = False
227 if rebuildMedia:
228 newDisc = True
229 logger.debug("Setting new disc flag based on rebuildMedia flag.")
230 else:
231 if blankBehavior is None:
232 logger.debug("Default media blanking behavior is in effect.")
233 if todayIsStart:
234 newDisc = True
235 logger.debug("Setting new disc flag based on todayIsStart.")
236 else:
237
238 logger.debug("Optimized media blanking behavior is in effect based on configuration.")
239 if blankBehavior.blankMode == "daily" or (blankBehavior.blankMode == "weekly" and todayIsStart):
240 logger.debug("New disc flag will be set based on blank factor calculation.")
241 blankFactor = float(blankBehavior.blankFactor)
242 logger.debug("Configured blanking factor: %.2f", blankFactor)
243 available = writer.retrieveCapacity().bytesAvailable
244 logger.debug("Bytes available: %s", displayBytes(available))
245 required = writer.getEstimatedImageSize()
246 logger.debug("Bytes required: %s", displayBytes(required))
247 ratio = available / (1.0 + required)
248 logger.debug("Calculated ratio: %.2f", ratio)
249 newDisc = (ratio <= blankFactor)
250 logger.debug("%.2f <= %.2f ? %s", ratio, blankFactor, newDisc)
251 else:
252 logger.debug("No blank factor calculation is required based on configuration.")
253 logger.debug("New disc flag [%s].", newDisc)
254 return newDisc
255
256
257
258
259
260
262 """
263 Writes a store indicator file into staging directories.
264
265 The store indicator is written into each of the staging directories when
266 either a store or rebuild action has written the staging directory to disc.
267
268 @param config: Config object.
269 @param stagingDirs: Dictionary mapping directory path to date suffix.
270 """
271 for stagingDir in list(stagingDirs.keys()):
272 writeIndicatorFile(stagingDir, STORE_INDICATOR,
273 config.options.backupUser,
274 config.options.backupGroup)
275
276
277
278
279
280
282 """
283 Runs a consistency check against media in the backup device.
284
285 It seems that sometimes, it's possible to create a corrupted multisession
286 disc (i.e. one that cannot be read) although no errors were encountered
287 while writing the disc. This consistency check makes sure that the data
288 read from disc matches the data that was used to create the disc.
289
290 The function mounts the device at a temporary mount point in the working
291 directory, and then compares the indicated staging directories in the
292 staging directory and on the media. The comparison is done via
293 functionality in C{filesystem.py}.
294
295 If no exceptions are thrown, there were no problems with the consistency
296 check. A positive confirmation of "no problems" is also written to the log
297 with C{info} priority.
298
299 @warning: The implementation of this function is very UNIX-specific.
300
301 @param config: Config object.
302 @param stagingDirs: Dictionary mapping directory path to date suffix.
303
304 @raise ValueError: If the two directories are not equivalent.
305 @raise IOError: If there is a problem working with the media.
306 """
307 logger.debug("Running consistency check.")
308 mountPoint = tempfile.mkdtemp(dir=config.options.workingDir)
309 try:
310 mount(config.store.devicePath, mountPoint, "iso9660")
311 for stagingDir in list(stagingDirs.keys()):
312 discDir = os.path.join(mountPoint, stagingDirs[stagingDir])
313 logger.debug("Checking [%s] vs. [%s].", stagingDir, discDir)
314 compareContents(stagingDir, discDir, verbose=True)
315 logger.info("Consistency check completed for [%s]. No problems found.", stagingDir)
316 finally:
317 unmount(mountPoint, True, 5, 1)
318
319
320
321
322
323
324
325
326
327
329 """
330 Finds the correct daily staging directory to be written to disk.
331
332 In Cedar Backup v1.0, we assumed that the correct staging directory matched
333 the current date. However, that has problems. In particular, it breaks
334 down if collect is on one side of midnite and stage is on the other, or if
335 certain processes span midnite.
336
337 For v2.0, I'm trying to be smarter. I'll first check the current day. If
338 that directory is found, it's good enough. If it's not found, I'll look for
339 a valid directory from the day before or day after I{which has not yet been
340 staged, according to the stage indicator file}. The first one I find, I'll
341 use. If I use a directory other than for the current day I{and}
342 C{config.store.warnMidnite} is set, a warning will be put in the log.
343
344 There is one exception to this rule. If the C{options.full} flag is set,
345 then the special "span midnite" logic will be disabled and any existing
346 store indicator will be ignored. I did this because I think that most users
347 who run C{cback3 --full store} twice in a row expect the command to generate
348 two identical discs. With the other rule in place, running that command
349 twice in a row could result in an error ("no unstored directory exists") or
350 could even cause a completely unexpected directory to be written to disc (if
351 some previous day's contents had not yet been written).
352
353 @note: This code is probably longer and more verbose than it needs to be,
354 but at least it's straightforward.
355
356 @param options: Options object.
357 @param config: Config object.
358
359 @return: Correct staging dir, as a dict mapping directory to date suffix.
360 @raise IOError: If the staging directory cannot be found.
361 """
362 oneDay = datetime.timedelta(days=1)
363 today = datetime.date.today()
364 yesterday = today - oneDay
365 tomorrow = today + oneDay
366 todayDate = today.strftime(DIR_TIME_FORMAT)
367 yesterdayDate = yesterday.strftime(DIR_TIME_FORMAT)
368 tomorrowDate = tomorrow.strftime(DIR_TIME_FORMAT)
369 todayPath = os.path.join(config.stage.targetDir, todayDate)
370 yesterdayPath = os.path.join(config.stage.targetDir, yesterdayDate)
371 tomorrowPath = os.path.join(config.stage.targetDir, tomorrowDate)
372 todayStageInd = os.path.join(todayPath, STAGE_INDICATOR)
373 yesterdayStageInd = os.path.join(yesterdayPath, STAGE_INDICATOR)
374 tomorrowStageInd = os.path.join(tomorrowPath, STAGE_INDICATOR)
375 todayStoreInd = os.path.join(todayPath, STORE_INDICATOR)
376 yesterdayStoreInd = os.path.join(yesterdayPath, STORE_INDICATOR)
377 tomorrowStoreInd = os.path.join(tomorrowPath, STORE_INDICATOR)
378 if options.full:
379 if os.path.isdir(todayPath) and os.path.exists(todayStageInd):
380 logger.info("Store process will use current day's stage directory [%s]", todayPath)
381 return { todayPath:todayDate }
382 raise IOError("Unable to find staging directory to store (only tried today due to full option).")
383 else:
384 if os.path.isdir(todayPath) and os.path.exists(todayStageInd) and not os.path.exists(todayStoreInd):
385 logger.info("Store process will use current day's stage directory [%s]", todayPath)
386 return { todayPath:todayDate }
387 elif os.path.isdir(yesterdayPath) and os.path.exists(yesterdayStageInd) and not os.path.exists(yesterdayStoreInd):
388 logger.info("Store process will use previous day's stage directory [%s]", yesterdayPath)
389 if config.store.warnMidnite:
390 logger.warning("Warning: store process crossed midnite boundary to find data.")
391 return { yesterdayPath:yesterdayDate }
392 elif os.path.isdir(tomorrowPath) and os.path.exists(tomorrowStageInd) and not os.path.exists(tomorrowStoreInd):
393 logger.info("Store process will use next day's stage directory [%s]", tomorrowPath)
394 if config.store.warnMidnite:
395 logger.warning("Warning: store process crossed midnite boundary to find data.")
396 return { tomorrowPath:tomorrowDate }
397 raise IOError("Unable to find unused staging directory to store (tried today, yesterday, tomorrow).")
398