From a22d7770b5c083c375ade9ae74e6ab717e19ea69 Mon Sep 17 00:00:00 2001 From: gm Date: Thu, 1 Aug 2019 07:25:54 +0200 Subject: [PATCH] Swap create_plugin with new Python code --- scripts/create_plugin | 176 ++++++++++++++++++++------------------- scripts/create_plugin.py | 107 ------------------------ scripts/dpkg_status.py | 4 +- 3 files changed, 91 insertions(+), 196 deletions(-) delete mode 100755 scripts/create_plugin.py diff --git a/scripts/create_plugin b/scripts/create_plugin index 24eeb7f..413a0d1 100755 --- a/scripts/create_plugin +++ b/scripts/create_plugin @@ -1,4 +1,4 @@ -#!/bin/bash -e +#!/usr/bin/python3 # Copyright (C) 2018 Gunter Miegel coinboot.io # @@ -17,89 +17,91 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -EXCLUDE='^\/(proc|sys|dev|run|tmp|vagrant|var\/log|usr\/share\/dbus-1\/system-services|var\/lib/\dpkg\/[^info]|var\/cache).*' -OUTFILE=/tmp/plugin_file_list -OUTFILE_FINAL=/tmp/plugin_file_list_final -DPKG_STATUS=/var/lib/dpkg/status -INITIAL_DPKG_STATUS=/tmp/initial_status -FINAL_DPKG_STATUS=/tmp/dpkg_status -PID_FILE=/tmp/plugin_create_pid - -function start { - # Inside a Docker/runc container '/proc' is read-only, so we are skipping this step. - # Running outside a container can be identified by the entries of /proc/1/cgroup ending with plain ':/'. - if grep -qP 'cpu.*:\/$' /proc/1/cgroup; then - echo "200000" | sudo tee /proc/sys/fs/inotify/max_user_watches - fi - - sudo rm -fv $OUTFILE $OUTFILE_FINAL $INITIAL_DPKG_STATUS $FINAL_DPKG_STATUS $PID_FILE - - # TODO Also track deleted files - # According to strace is looks like --exclude is not preventing building the - # "watches" for the excluded directories but filters all file system events on them. - if [ $(which inotifywait) ]; then - sudo rm -f $OUTFILE - nohup sudo inotifywait -m -r --format '%w%f' -e create -e attrib -e moved_from -e moved_to -e modify --exclude $EXCLUDE -o $OUTFILE / > /tmp/plugin.log 2>&1 & - else - echo 'inotifywait is missing.' - echo 'Please install the package inotify-tools' - exit - fi - - PID=$! - - echo "PID of the process watch for file changes is: $PID" - - echo $PID > $PID_FILE - - # Inotifywait takes some time to build up the recursive file watching on - # / - so we have to wait and verify that we only proceed after the prepations - # are done. - while ! grep -q 'Watches established.' /tmp/plugin.log; do - echo -ne 'Waiting for file watching to be established...\r' - sleep 0.2 - done - - cp $DPKG_STATUS $INITIAL_DPKG_STATUS -} - -function finish { - plugin_name=$1 - - sudo kill $(cat /tmp/plugin_create_pid) - - dpkg_status.py --old $INITIAL_DPKG_STATUS --new $DPKG_STATUS --diff > $FINAL_DPKG_STATUS - - # Test all collected files if they really exists and if so write them - # to a final file list. - # Finally add the dpkg_status file we create beforehand. - while read path; do - if [ -f "$path" ]; then - echo $path - echo $path >> "$OUTFILE_FINAL" - fi - done < <(sort -u $OUTFILE) - - echo $FINAL_DPKG_STATUS >> $OUTFILE_FINAL - - tar -czvf $plugin_name.tar.gz -P -T $OUTFILE_FINAL -} - -function upload { - filepath=$1 - url=$2 - filename=$(basename $filepath) - curl -F "file=@$filepath;filename=$filename" $url -} - -case "$1" in - 'start') - start - ;; - 'finish') - finish $2 - ;; - 'upload') - upload $2 $3 - ;; -esac +"""Create Coinboot Plugins + +Usage: + create_plugin start + create_plugin finish + +Options: + -h --help Show this screen. + +""" + +import os +import tarfile +import re +from subprocess import call +from docopt import docopt + +DPKG_STATUS = '/var/lib/dpkg/status' +INITIAL_DPKG_STATUS = '/tmp/initial_status' +FINAL_DPKG_STATUS = '/tmp/dpkg_status' +PLUGIN_DIR = '/mnt/plugin/rootfs' + +EXCLUDE = ('/dev/', + '/proc/', + '/run/', + '/sys/', + '/tmp/', + '/usr/src', + '/usr/include', + '/usr/share/dbus-1/system-services', + '/vagrant', + '/var/cache', + '/var/lib/apt/lists', + '/var/lib/dpkg/[^info]', + '/var/log', + '.*__pycache__.*', + '.wget-hsts' + ) + + +def find(path_to_walk): + """Return results similar to the Unix find command run without options + i.e. traverse a directory tree and return all the file paths + """ + return [os.path.join(path, file) + for (path, dirs, files) in os.walk(path_to_walk) + for file in files] + +def main(arguments): + #print(arguments) + if arguments['start']: + call(['cp', '-v', DPKG_STATUS, INITIAL_DPKG_STATUS]) + elif arguments['finish']: + f = open(FINAL_DPKG_STATUS, 'w') + call(['dpkg_status.py', '--old', INITIAL_DPKG_STATUS, '--new', DPKG_STATUS, '--diff'], stdout=f) + + files_for_plugin_archive = [] + + for path in find(PLUGIN_DIR): + cleaned_path = re.sub(PLUGIN_DIR, '', path) + # FIXME: Switch to re.match() against path without PLUGIN_DIR prefix + if any(re.findall(pattern, cleaned_path) for pattern in EXCLUDE): + print('Excluded:', cleaned_path) + else: + print('Included:', cleaned_path) + files_for_plugin_archive.append(cleaned_path) + + files_for_plugin_archive.append(FINAL_DPKG_STATUS) + + tar = tarfile.open(arguments[''] + ".tar.gz", "w:gz") + for path in files_for_plugin_archive: + # If a file was deleted which was in the lower directory + # a whiteout file is created in the upper directory. + # So we don't can look at the upper director to track the + # deletion of such files. Else we look if the file is there + # at the merged directory with 'os.path.exists()'. + if os.path.exists(path): + # We have to specfiy explictly the file name in + # the archive to get an absolute path wit a leading '/' + tar.add(path, arcname=path) + else: + print('Whiteout file from lower dir:', path) + tar.close() + + +if __name__ == '__main__': + arguments = docopt(__doc__, version='Create Coinboot Plugins v0.1') + main(arguments) diff --git a/scripts/create_plugin.py b/scripts/create_plugin.py deleted file mode 100755 index 413a0d1..0000000 --- a/scripts/create_plugin.py +++ /dev/null @@ -1,107 +0,0 @@ -#!/usr/bin/python3 - -# Copyright (C) 2018 Gunter Miegel coinboot.io -# -# This file is part of Coinboot. -# -# Coinboot is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -"""Create Coinboot Plugins - -Usage: - create_plugin start - create_plugin finish - -Options: - -h --help Show this screen. - -""" - -import os -import tarfile -import re -from subprocess import call -from docopt import docopt - -DPKG_STATUS = '/var/lib/dpkg/status' -INITIAL_DPKG_STATUS = '/tmp/initial_status' -FINAL_DPKG_STATUS = '/tmp/dpkg_status' -PLUGIN_DIR = '/mnt/plugin/rootfs' - -EXCLUDE = ('/dev/', - '/proc/', - '/run/', - '/sys/', - '/tmp/', - '/usr/src', - '/usr/include', - '/usr/share/dbus-1/system-services', - '/vagrant', - '/var/cache', - '/var/lib/apt/lists', - '/var/lib/dpkg/[^info]', - '/var/log', - '.*__pycache__.*', - '.wget-hsts' - ) - - -def find(path_to_walk): - """Return results similar to the Unix find command run without options - i.e. traverse a directory tree and return all the file paths - """ - return [os.path.join(path, file) - for (path, dirs, files) in os.walk(path_to_walk) - for file in files] - -def main(arguments): - #print(arguments) - if arguments['start']: - call(['cp', '-v', DPKG_STATUS, INITIAL_DPKG_STATUS]) - elif arguments['finish']: - f = open(FINAL_DPKG_STATUS, 'w') - call(['dpkg_status.py', '--old', INITIAL_DPKG_STATUS, '--new', DPKG_STATUS, '--diff'], stdout=f) - - files_for_plugin_archive = [] - - for path in find(PLUGIN_DIR): - cleaned_path = re.sub(PLUGIN_DIR, '', path) - # FIXME: Switch to re.match() against path without PLUGIN_DIR prefix - if any(re.findall(pattern, cleaned_path) for pattern in EXCLUDE): - print('Excluded:', cleaned_path) - else: - print('Included:', cleaned_path) - files_for_plugin_archive.append(cleaned_path) - - files_for_plugin_archive.append(FINAL_DPKG_STATUS) - - tar = tarfile.open(arguments[''] + ".tar.gz", "w:gz") - for path in files_for_plugin_archive: - # If a file was deleted which was in the lower directory - # a whiteout file is created in the upper directory. - # So we don't can look at the upper director to track the - # deletion of such files. Else we look if the file is there - # at the merged directory with 'os.path.exists()'. - if os.path.exists(path): - # We have to specfiy explictly the file name in - # the archive to get an absolute path wit a leading '/' - tar.add(path, arcname=path) - else: - print('Whiteout file from lower dir:', path) - tar.close() - - -if __name__ == '__main__': - arguments = docopt(__doc__, version='Create Coinboot Plugins v0.1') - main(arguments) diff --git a/scripts/dpkg_status.py b/scripts/dpkg_status.py index 4ae5d3c..f49be66 100755 --- a/scripts/dpkg_status.py +++ b/scripts/dpkg_status.py @@ -22,7 +22,8 @@ between both on package section level. It can also merge both files to a union set with no redudant packages sections. -The purpose of this is to track status changes of DPKG between two points in time in a separate file. +The purpose of this is to track status changes of DPKG between two points in time +in a separate file. And it provides the abillity to merge that file into an existing DPKG status file. """ @@ -49,7 +50,6 @@ def print_dict(set_dict): Prints the package text blocks from a dict. """ for package_block in set_dict: - #print(set_dict[package_block] + '\n') print(set_dict[package_block] + '\n') def main():