forked from evanpurkhiser/dots
-
Notifications
You must be signed in to change notification settings - Fork 0
/
dots
executable file
·633 lines (495 loc) · 24.3 KB
/
dots
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
#!/usr/bin/env python
#
# dots - A cascading dot file management tool
#
# The dots utility is used to manage a grouping of dotfiles for various
# environments and UNIX like platforms. The primary goal of this utility it to
# facilitate the logical separation of dotfiles and providing the ability to
# 'compile' this organization into a usable directory tree of files.
#
# DIRECTORY STRUCTURE AND GROUPINGS
#
# By default configuration files should be stored two subdirectories deep from
# the location where this script is located. For example `machines/desktop/`
# could be used to store configuration files specific to a users desktop
# machine. The exception to this is the `base` group, which is only one
# directory deep.
#
# You may have as many configuration groupings as you would like.
#
# CASCADING OF CONFIGURATION FILES
#
# If two groups are specified for a configuration environment that both contain
# the same configuration file, then the file will 'cascade'. What this means is
# that the file in the second group will be appended to the first group.
#
# OVERRIDING AND EXTENDING
#
# The cascading logic can be modified by 'overriding' or 'extending' a file.
#
# Overriding a file allows you to force configuration files that exist earlier
# in the configuration environment groups list to be discarded instead of being
# used as the base for a configuration file to be appended to. For example:
#
# base/bash/bashrc
# machines/desktop/bash/bashrc.override
#
# groups = ['base', 'machines/desktop']
#
# Instead of the machines/desktop version of the bashrc file being amended to
# the base bashrc file, it will simply discard the base version of the file.
#
# Extending a file allows you to specify 'explicit append points' in a file.
# This way instead of having a cascading file be appended to the end, it could
# be appended somewhere else. Continuing from the last example (without the
# .override):
#
# If the `base/bash/bashrc` file contains the symbol `!!@@' in the file, this is
# where the `machines/desktop/bash/bashrc` file would be appended.
#
# You may also use 'named append points'. This allows you to insert 'fragment'
# files into a file. For example, say we have the following files:
#
# base/bash/bashrc
# base/bash/bashrc.aliases
#
# If somewhere within the `base/bash/bashrc` file we have the symbol
# '!!@@aliases' then the `bashrc.aliases` file would be appended at.
#
# Fragment files are subject to all of the same logic described above
# (extending, append points, and even named append points)
#
# INSTALLATION SCRIPTS
#
# Each configuration file may have an associated .install script. This is a
# script that will be executed when the configuration file is installed.
# Remember that configuration files are only installed if they differ from the
# currently installed configuration file. So install scripts will only be run on
# updates or first installs of the configuration file.
#
# PROGRAM USAGE
#
# See USAGE variable
from __future__ import print_function
import sys
import os
import shutil
import subprocess
import glob
import re
import tempfile
import argparse
import hashlib
import warnings
USAGE="""dots - A dot file pre-processor and installer
dots [-c CONFIG] COMMAND [options]
The following commands and options are available:
-c Specify the file to load and save configuration environment groups
lists to. If not specified the default configuration file will be used
groups Manage available configuration groups
known List all configuration groups that are available to be
specified for a configuration environment list
current Get the current configuration environment groups
set Specify a space separated list of valid configuration groups
to enable for this environment
add Add one or more groups to the configuration file
del Removes one or more groups from the configuration file
clear Remove all current groups from the configuration file
diff Get the difference between the source files, and currently installed
configuration files. This should be though of as a representation of
'what will change' when you install. This will use git-diff to do the
diffing, thus you may pass any valid git-diff options such as
--shortstat
[PATHS] A list of one or more files or paths to check. If this is
omitted then the entire tree of source configuration files
will be diffed
files List out all of the currently tracked configuration files for the
currently set configration groups
[PATHS] A list of one or more paths to filter the list by
install Compile and install one or more configuration files into a location.
If no specific paths are specified then the entire source tree will
be installed into the specified location and after-install scripts
will be executed
[PATHS] A list of one or more files or paths to install into the
specified location. If omitted the entire tree will be
installed
-l Specify the location to install the configuration files to.
If omitted the default XDG_CONFIG_HOME path will be used
-r Reinstall all config files and execute all install scripts.
help This help message"""
# Get the location that we will be installing all of the files into
INSTALL_DIR = os.getenv('XDG_CONFIG_HOME', os.path.join(os.environ['HOME'], '.config'))
# Locate the configuration directory that all source configuration files are
# stored in. For now we will install them into $HOME/.local/etc
SOURCE_DIR = os.path.join(os.environ['HOME'], '.local/etc')
# The named append point identifier in files
AP_IDENTIFIER = '!!@@'
# The file extension used to identify 'overriding' files
OVERRIDE_EXT = '.override'
# The filename extension to use for after-install scripts
INSTALL_SCRIPT_EXT = '.install'
# Globally handle exceptions in a clean way
def handle_exception(exc_type, value, trace):
# Normal exception trace in debug mode
if os.environ.get('DOTS_DEBUG'):
return sys.__excepthook__(exc_type, value, trace)
# print exception message to stderr
sys.stderr.write("\033[91m{}\033[0m\n".format(value))
sys.excepthook = handle_exception
# Globaly handle warnings in a clean way
def handle_warning(message, category, filename, line):
sys.stderr.write("\033[93m{}: {}\033[0m\n".format(category.__name__, message))
warnings.showwarning = handle_warning
# Helper function to execute a git diff process
def git_diff(git_args, path1, path2):
subprocess.call(['git', 'diff', '--diff-filter=MA'] + git_args + [path1, path2])
class Configuration(object):
# This is the default groups file that the configuration group listing for
# the machine should be stored in
default_group_file = os.path.join(INSTALL_DIR, 'dots', 'config-groups')
def __init__(self, group_file=default_group_file, groups=[]):
# This is a list of groups that are only a single directory deep and do not
# contain any groups in sub-directories. The directories in these folders
# will be the directories that are installed as configuration files
single_directory_groups = ['base']
# Determine which groups are valid by looking at the directory structure of
# that the script is installed in. Groups not defined in the
# single_directory_groups array will be considered nested groups
valid_groups = [g[len(SOURCE_DIR) + 1:] for g in glob.glob(os.path.join(SOURCE_DIR, '*/*'))]
valid_groups = [g for g in valid_groups if not g.startswith(tuple(single_directory_groups))]
self.valid_groups = valid_groups + single_directory_groups
self.group_file = group_file
self.groups = groups
@property
def groups(self):
"""Return the list of groups configured for this system"""
return self._groups
@groups.setter
def groups(self, groups):
"""Set the list of groups for this configuration"""
# Trim trailing and leading slashes
groups = [x.strip('/') for x in groups]
# Warn for invalid groups
missing_groups = set(groups).difference(self.valid_groups)
if missing_groups:
warnings.warn("Configuration group does not exist: {}".format(", ".join(missing_groups)), RuntimeWarning)
self._groups = [x for x in groups if x in self.valid_groups]
return groups
def load_from_file(self):
"""Load groups from the configuration groups file
This will complain if one of the groups specified in the file doesn't
exist in the source tree (meaning it's considered an 'invalid group'"""
if not os.path.exists(self.group_file): return
with open(self.group_file, 'r') as f:
self.groups = f.read().splitlines()
if not self.groups:
warnings.warn("No configuration groups are currently enabled.", RuntimeWarning)
def save_to_file(self):
"""This will save the currently defined list of groups for this configuration
into the groups file"""
directory = os.path.dirname(self.group_file)
# Ensure the configuration group file directory exists
if not os.path.exists(directory):
os.makedirs(directory)
with open(self.group_file, 'w+') as f:
f.write('\n'.join(self.groups))
def files(self, path_filters=None):
"""Get all configuration file objects that are to be installed"""
files_set = set()
for group in self.groups:
group_path=os.path.join(SOURCE_DIR, group)
for root,_, files in os.walk(group_path):
# Trim the group name from the directory path
root = root[len(group_path) + 1:]
for file in files:
# Remove the override extension. This will is going to be
# installed without this extension, ConfigFile.real_paths
# will handle locating this file
if file.endswith(OVERRIDE_EXT):
file = file[:-len(OVERRIDE_EXT)]
# Ignore install scripts
if file.endswith(INSTALL_SCRIPT_EXT):
continue
files_set.add(os.path.join(root, file))
# Filter out files not in the path_filters
if path_filters:
# Create a copy of path_filters as if each is a directory
dir_path_filters = [os.path.normpath(p) + os.sep for p in path_filters]
# Filter out files that aren't in the files / dirs list
files_set = filter(lambda p:
p in path_filters or list(filter(p.startswith, dir_path_filters)),
files_set)
# Get files as ConfigFile objects
files = [ConfigFile(file, self) for file in files_set]
# Don't include named_fragment files
return [f for f in files if not f.is_named_fragment()]
def install_tree(self, to=None, path_filters=None, exec_install_scripts=False, reinstall=False):
"""Install the configuration tree into a specific directory"""
if not to: to = INSTALL_DIR
# Install all files to the location
[f.install(to, exec_install_scripts, reinstall) for f in self.files(path_filters)]
def diff_installed(self, git_args=[], path_filters=None):
"""Run a diff against the currently installed tree. This uses git as an
external command to do the diffing. If a list of file paths are passed
in then only those files will be diffed"""
temp_dir = tempfile.mkdtemp()
self.install_tree(temp_dir, path_filters)
git_diff(git_args, INSTALL_DIR, temp_dir)
shutil.rmtree(temp_dir)
class ConfigFile(object):
@staticmethod
def trim_file_whitespace(file_lines):
"""Remove whiteface elements from the beginning and ends of a list"""
for iterator in [enumerate(file_lines), reversed(list(enumerate(file_lines)))]:
for i, line in iterator:
if line.isspace():
file_lines[i] = None
else: break
# Trim front and back empty lines
return [f for f in file_lines if f]
@staticmethod
def strip_shebang(file_lines):
"""Removes the shebang from the first line of a list"""
if file_lines[0].startswith("#!/"):
del file_lines[0]
file_lines = ConfigFile.trim_file_whitespace(file_lines)
return file_lines
@staticmethod
def insert_at_ap(file_lines, insert_lines, ap_name=''):
"""Insert a list of lines into the file_lines at a given append point.
This takes whitespace into account and will add the whitespace in front
of all inserted lines"""
# Locate the append point line (ignoring whitespace). This will throw an
# exception if the append point cannot be found. This should be expected
slice_at = [l.strip() for l in file_lines].index(AP_IDENTIFIER + ap_name)
ws = re.match('\s*', file_lines[slice_at]).group()
# Splice the inserted lines in (including whitespace)
del file_lines[slice_at]
file_lines[slice_at:1] = [l if l == '\n' else ws + l for l in insert_lines]
return file_lines
def __init__(self, relative_path, config):
self.path = relative_path
self.config = config
self.compiled = None
def name(self):
"""Get the name of the file"""
return os.path.basename(self.path)
def directory(self):
"""Get the directory the file is located in"""
return os.path.dirname(self.path)
def real_paths(self):
"""Get the real paths of the source file"""
paths = [os.path.join(SOURCE_DIR, group, self.path) for group in self.config.groups]
# Check for overriding files and remove previous paths accordingly
# We must iterate in reverse since the paths list starts at the file in
# the lowest priority group
for i, path in reversed(list(enumerate(paths))):
# Skip if there is no overriding file in this group
if not os.path.exists(path + OVERRIDE_EXT): continue
# Use the overriding file and ignore all previous paths
paths[i] = path + OVERRIDE_EXT
paths = paths[i:]
return [p for p in paths if os.path.isfile(p)]
def mode(self):
"""Determine the mode of this file. If the file has multiple overriding
files that don't have matching permissions, then the highest mode will
be used"""
return max([os.stat(p).st_mode for p in self.real_paths()])
def named_append_points(self):
""" Get a list of named append points in this file that will
include fragments"""
append_points = set()
for path in self.real_paths():
with open(path, 'r') as file:
for line in file:
line = line.strip()
if line.startswith(AP_IDENTIFIER):
append_points.add(line[len(AP_IDENTIFIER):])
return filter(None, append_points)
def is_named_fragment(self):
"""Check if this configuration file is actually a named fragment for
another file"""
match = re.match('^(.+)\.(.+)$', self.name())
if not match:
return False
# Get the target configuration file
target_path = os.path.join(self.directory(), match.group(1))
target_file = ConfigFile(target_path, self.config)
# The named append point to look for
ap_name = match.group(2)
# Check if the target file has the append point
return match.group(2) in target_file.named_append_points()
def compile(self):
"""Compile this configuration file. This will take each file from the
group and merge it based on these rules:
1. Files will be merged down from lowest priority groups (specified
first) to highest priority (specified last)
2. If the file being merged into includes an explicit append point it
will be merged at that location. Else it will just be appended to
the end of the file being merged into.
3. If the file has named append points then we will look for a fragment
file that matches the append point name, if we can find it it will be
merged in."""
self.compiled = None
compiling_files = []
# Read all file paths into arrays
# This may be memory inefficient and may have to be changed later
for path in self.real_paths():
with open(path, 'r') as file:
contents = file.readlines()
# Replace front and back empty lines with None
contents = self.trim_file_whitespace(contents)
# Ensure a trailing newline
if contents and not contents[-1].endswith('\n'):
contents[-1] += '\n'
# Ensure the file only has one explicit default append point
if [p.strip() for p in contents].count(AP_IDENTIFIER) > 1:
raise Exception("More than one explicit append point in {0}".format(path))
compiling_files.append(contents)
# Nothing left to do if there were no files
if not compiling_files:
self.compiled_file = ""
return self
compiled = compiling_files[0]
# Handle merging the file_data into one file. This will look for default
# Append points, but if it can't find any will default to appending the file
for single_file in compiling_files[1:]:
# Remove shebang from first line
single_file = self.strip_shebang(single_file)
# Check if we need to slice into the array to insert
try:
compiled = self.insert_at_ap(compiled, single_file)
except ValueError:
compiled += ['\n'] + single_file
# Handle merging in the named append points
for name in self.named_append_points():
appending_file = ConfigFile(self.path + '.' + name, self.config)
appending_data = appending_file.compile().compiled
# Replace all instances of this named append point
while True:
try:
compiled = self.insert_at_ap(compiled, appending_data, name)
except:
break
# Remove unused append point identifiers
self.compiled = [l for l in compiled if not l.strip().startswith(AP_IDENTIFIER)]
return self
def diff(self, git_args=[], against=INSTALL_DIR):
"""Display a diff of the compiled file contest against the currently
installed version of this file. This uses git as an external command to
do the diffing"""
if self.compiled is None: self.compile()
# Create a named temporary file to allow git to diff
with tempfile.NamedTemporaryFile('w+') as file:
file.writelines(self.compiled)
file.flush()
git_diff(git_args, os.path.join(against, self.path), file.name)
def exec_install_scripts(self, install_directory):
"""Execute all install scripts associated with this configuration
file. The execution will happen in the installation directory containing
the installed file"""
# Locate all installation scripts
paths = [os.path.join(SOURCE_DIR, group, self.path) + INSTALL_SCRIPT_EXT
for group in self.config.groups]
# Get the current working directory to execute the script in
cwd = os.path.join(install_directory, os.path.dirname(self.path))
# Setup some additiona environment variables for the scripts
environ = os.environ
environ.update({"DOTS_SOURCE": SOURCE_DIR})
# Execute install scripts that exist
for script in [p for p in paths if os.path.exists(p)]:
subprocess.Popen(script, cwd=cwd, env=environ).wait()
def install(self, to, exec_install_scripts=False, reinstall=False):
"""Install this file to a location"""
if not self.compiled: self.compile()
# Ensure directory exists for file
path = os.path.join(to, self.path)
try:
os.makedirs(os.path.dirname(path))
except:
pass
# Test that the file hasn't changed
if os.path.exists(path):
installed = hashlib.md5(open(path).read()).digest()
compiled = hashlib.md5(''.join(self.compiled)).digest()
if not reinstall and installed == compiled and os.stat(path).st_mode == self.mode():
return
# Write the file
with open(path, 'w') as file:
file.writelines(self.compiled)
# Set the file's mode
os.chmod(path, self.mode())
# Run install scripts
if exec_install_scripts: self.exec_install_scripts(to)
def setup_argparser():
parser = argparse.ArgumentParser(add_help=False)
# Handle getting the configuration path
parser.add_argument('-c', '--config', default=Configuration.default_group_file)
# Setup the sub-commands
sub_commands = parser.add_subparsers(dest='command')
sub_commands.required = True
# The main sub commands
groups = sub_commands.add_parser('groups', add_help=False)
diff = sub_commands.add_parser('diff', add_help=False)
files = sub_commands.add_parser('files', add_help=False)
install = sub_commands.add_parser('install', add_help=False)
helps = sub_commands.add_parser('help', add_help=False)
# Groups sub commands
groups_commands = groups.add_subparsers(dest='group_command')
groups_known = groups_commands.add_parser('known', add_help=False)
groups_current = groups_commands.add_parser('current', add_help=False)
groups_clear = groups_commands.add_parser('clear', add_help=False)
groups_set = groups_commands.add_parser('set', add_help=False)
groups_add = groups_commands.add_parser('add', add_help=False)
groups_del = groups_commands.add_parser('del', add_help=False)
# Groups-set options
groups_set.add_argument('group', nargs='+')
groups_add.add_argument('group', nargs='+')
groups_del.add_argument('group', nargs='+')
# Diff options
diff.add_argument('file', nargs='*')
# Files options
files.add_argument('file', nargs='*')
# Install options
install.add_argument('-l', '--location', default=INSTALL_DIR)
install.add_argument('file', nargs='*')
install.add_argument('-r', '--reinstall', action='store_true')
return parser
if __name__ == '__main__':
args, extra_args = setup_argparser().parse_known_args()
config = Configuration(group_file=args.config)
config.load_from_file()
if args.command == 'groups':
if args.group_command == 'known':
print('\n'.join(config.valid_groups + ['']), end='')
elif args.group_command == 'current':
print('\n'.join(config.groups + ['']), end='')
elif args.group_command == 'set':
config.groups = args.group
config.save_to_file()
elif args.group_command == 'add':
for group in args.group:
if group in config.groups:
warnings.warn("Group {0} is already in current groups. I'll ignore this group!.".format(group), RuntimeWarning)
args.group.remove(group)
config.groups = config.groups + args.group
config.save_to_file()
elif args.group_command == 'del':
for group in args.group:
if group in config.groups:
config.groups.remove(group)
else:
warnings.warn("Group {0} wasn't in current groups. I'll ignore this group!".format(group), RuntimeWarning)
config.save_to_file()
elif args.group_command == 'clear':
config.groups = []
config.save_to_file()
elif args.command == 'diff':
config.diff_installed(extra_args, args.file)
elif args.command == 'files':
files = [f.path for f in config.files(args.file)]
for file in sorted(files): print(file)
elif args.command == 'install':
config.install_tree(args.location, args.file, True, args.reinstall)
elif args.command == 'help':
print(USAGE)