From e2ffeda501cb254dc8f6b0a600bbbe5862807920 Mon Sep 17 00:00:00 2001 From: Thor Harald Johansen Date: Mon, 1 Jun 2020 13:45:23 +0200 Subject: [PATCH] Fix problems with shutdown, REPL, etc. --- .env | 1 + .gitmodules | 3 + .vscode/launch.json | 21 ++++++ src/zasd/__main__.py | 3 + src/zasd/{__init__.py => entrypoint.py} | 67 +++++++++-------- src/zasd/filesystem.py | 8 +-- src/zasd/filesystems/__init__.py | 0 src/{ => zasd/filesystems}/test.py | 2 +- src/zasd/filesystems/zfs.py | 51 +++---------- .../filesystems/{zfs_old.py => zfs_old.txt} | 0 src/zasd/repl.py | 71 +++++++++++-------- third_party/packages/six | 1 + vs.code-workspace | 4 +- 13 files changed, 122 insertions(+), 110 deletions(-) create mode 100644 .env create mode 100644 .vscode/launch.json create mode 100644 src/zasd/__main__.py rename src/zasd/{__init__.py => entrypoint.py} (76%) delete mode 100644 src/zasd/filesystems/__init__.py rename src/{ => zasd/filesystems}/test.py (81%) rename src/zasd/filesystems/{zfs_old.py => zfs_old.txt} (100%) create mode 160000 third_party/packages/six diff --git a/.env b/.env new file mode 100644 index 0000000..b8a4a5a --- /dev/null +++ b/.env @@ -0,0 +1 @@ +PYTHONPATH=./src:./third_party/modules \ No newline at end of file diff --git a/.gitmodules b/.gitmodules index 0991353..10a9e0d 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,3 +4,6 @@ [submodule "third_party/packages/ptpython"] path = third_party/packages/ptpython url = https://github.com/prompt-toolkit/ptpython.git +[submodule "third_party/packages/six"] + path = third_party/packages/six + url = https://github.com/benjaminp/six.git diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..7795f98 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,21 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "Python: Current File", + "type": "python", + "request": "launch", + "program": "${file}", + "console": "integratedTerminal" + }, + { + "name": "Python: Module", + "type": "python", + "request": "launch", + "module": "zasd" + } + ] +} \ No newline at end of file diff --git a/src/zasd/__main__.py b/src/zasd/__main__.py new file mode 100644 index 0000000..79a961a --- /dev/null +++ b/src/zasd/__main__.py @@ -0,0 +1,3 @@ +from zasd.entrypoint import start + +start() \ No newline at end of file diff --git a/src/zasd/__init__.py b/src/zasd/entrypoint.py similarity index 76% rename from src/zasd/__init__.py rename to src/zasd/entrypoint.py index 02f0836..35a1e32 100644 --- a/src/zasd/__init__.py +++ b/src/zasd/entrypoint.py @@ -1,68 +1,71 @@ -from sys import stdout, stderr -from os import isatty - +import sys +from sys import stdout, stderr import signal import time from functools import partial, reduce from itertools import islice import asyncio +from asyncio import get_event_loop from subprocess import run, PIPE from datetime import datetime, timezone, timedelta -from apscheduler.triggers.cron import CronTrigger # type: ignore -from apscheduler.triggers.interval import IntervalTrigger # type: ignore +from apscheduler.triggers.cron import CronTrigger # type: ignore +from apscheduler.triggers.interval import IntervalTrigger # type: ignore from zasd.apsched import AsyncIOPriorityScheduler, \ - AsyncIOPriorityExecutor + AsyncIOPriorityExecutor from zasd.config.loader import load_config import zasd.config as config -from zasd.filesystems.zfs import ZFSFilesystem from zasd.filesystem import FilesystemRegistry +from zasd.filesystems.zfs import ZFSFilesystem from zasd.log import configure_logging, log from zasd.repl import repl async def main(): - #event_loop.add_signal_handler( - # signal.SIGINT, partial(signal_handler, 'SIGINT')) - #event_loop.add_signal_handler( - # signal.SIGTERM, partial(signal_handler, 'SIGTERM')) + main.task = asyncio.current_task() + + log.info('sys.path = %s', sys.path) await load_config() configure_logging() + if stdout.isatty(): + asyncio.create_task(repl()) + log.info('Processing jobs') # Load and activate snapshot schedules - #load_schedules() - #scheduler.start() - - #if isatty(): - # event_loop.create_task(spinner) + #launch_scheduler() -event_loop = asyncio.get_event_loop() + try: + while True: + await asyncio.sleep(3600) + except asyncio.CancelledError: + pass -scheduler = AsyncIOPriorityScheduler( - event_loop = event_loop, - executors = {'default': AsyncIOPriorityExecutor()}) +def start(): + try: + code = asyncio.get_event_loop().run_until_complete(main()) + finally: + print('Terminating') -asyncio.ensure_future(repl()) -asyncio.ensure_future(main()) - -try: - event_loop.run_forever() -finally: - log.info('Terminating') +def stop(): + main.task.cancel() def signal_handler(signame): log.info('Received %s', signame) - asyncio.get_event_loop().stop() + get_event_loop().stop() + +def launch_scheduler(): + scheduler = AsyncIOPriorityScheduler( + event_loop = get_event_loop(), + executors = {'default': AsyncIOPriorityExecutor()}) -def load_schedules(): for schedule in schedules(): if schedule['disabled']: continue - + tag = schedule['tag'] for fs in schedule['filesystems']: scheduler.add_job( @@ -81,6 +84,10 @@ def load_schedules(): id = 'destroy', group = 'destroy') + + #scheduler.start() + + # Retrieve all schedules and merge with default schedule def schedules(): return ({**config.get('defaults'), **dict(s)} diff --git a/src/zasd/filesystem.py b/src/zasd/filesystem.py index d40ea3f..cb05ec3 100644 --- a/src/zasd/filesystem.py +++ b/src/zasd/filesystem.py @@ -206,8 +206,6 @@ class FilesystemBase(ABC): values. ''' return hash(self) == hash(other) - - class DispatcherMixin(Generic[EventT]): ''' Class for dispatching filesystem events to asynchronous generators produced by a factory function. Events are consumed by iterating @@ -243,7 +241,7 @@ class DispatcherMixin(Generic[EventT]): class FilesystemRegistry: ''' Class that keeps a global registry of filesystem classes and labels ''' - _filesystems: MutableMapping[str, TypingType[FilesystemBase]] = {} + _filesystems: MutableMapping[str, FilesystemBase] = {} _labels: MutableSet[str] = set() def __init__(self): @@ -252,7 +250,7 @@ class FilesystemRegistry: @classmethod def filesystems(cls) -> Mapping[str, TypingType[FilesystemBase]]: ''' Retrieve a dictionary of keys to registered filesystem classes. ''' - return dict(cls._filesystems) + return cls._filesystems @classmethod def has(cls, key: str) -> bool: @@ -288,4 +286,4 @@ class FilesystemRegistry: filesystem becomes unavailable, or the configuration file was reloaded and a filesystem has disappeared from it. ''' assert label in cls._labels - cls._labels.remove(label) + cls._labels.remove(label) \ No newline at end of file diff --git a/src/zasd/filesystems/__init__.py b/src/zasd/filesystems/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/test.py b/src/zasd/filesystems/test.py similarity index 81% rename from src/test.py rename to src/zasd/filesystems/test.py index 4492b5f..17efd47 100644 --- a/src/test.py +++ b/src/zasd/filesystems/test.py @@ -8,5 +8,5 @@ import pprint #fs = ZFSFilesystem() #fs.label() -pprint.pprint(dict(os.environ), indent=2) +#pprint.pprint(dict(os.environ), indent=2) pprint.pprint(sys.path) diff --git a/src/zasd/filesystems/zfs.py b/src/zasd/filesystems/zfs.py index a9b6263..c0a937a 100644 --- a/src/zasd/filesystems/zfs.py +++ b/src/zasd/filesystems/zfs.py @@ -1,71 +1,40 @@ ''' ZFS implementation ''' -from typing import Any, Tuple, Set, Mapping, Callable +from typing import Any, Tuple, Set, Mapping, Callable -from ..filesystem import FilesystemRegistry, FilesystemBase, FilesystemT, \ - SnapshotKeyT, DispatcherMixin +from zasd.filesystem import FilesystemRegistry, FilesystemBase, FilesystemT, \ + SnapshotKeyT, DispatcherMixin class ZFSFilesystem(DispatcherMixin, FilesystemBase): ''' ZFS filesystem implementation. ''' @classmethod def key(cls) -> str: - ''' Retrieve the registry key for this filesystem class. ''' - - # Filesystem classes must add themselves to the filesystem registry as - # soon as they are imported. + return 'zfs' @classmethod async def filesystems(cls) -> Mapping[str, FilesystemT]: - ''' Retrieve a dictionary of labels to filesystems managed by this - filesystem class. ''' + return dict() def __init__(self): super().__init__() - @classmethod def configure(cls) -> Tuple[Mapping[str, Any], Callable[[Any], None]]: - ''' Retrieve tuple for configuring the filesystem. ''' + glo = dict() - # This method returns a tuple (global_scope, callback), consisting of - # a dictionary and a callback function. It is called each time a - # configuration file has loaded, before it is executed. - # - # The dictionary consists of variables to be deep copied and exposed - # on the global execution scope of the configuration file. This may be - # used by the Filesystem class to provide constants, functions and - # variables that the user will need in order to configure filesystems - # of this class. + def configure(options): + pass - # The callback is invoked as callback(options) once execution has - # completed. Options are retrived from a variable Filesystem.key() on - # the local execution scope of the configuration file and will equal - # None if no such variable was assigned by the user. The variable is - # typically a dictionary, but may be of any type, and may be used by - # the Filesystem class to retrieve configuration options the user has - # set in order to configure filesystems of this class. ''' + return (glo, configure) def label(self) -> str: ''' Retrieve the filesystem label.''' - # Labels must allow users to easily identify the underlying - # filesystems. Device names, drive letters, volume labels and - # mountpoints should be avoided, since these are not persistent and do - # not uniquely identify a filesystem. If the underlying filesystem - # does not provide labels that satisfy these criteria, a means for the - # user to map unique labels to unique filesystem identifiers must be - # provided in the configuration file. - async def children(self) -> Mapping[str, FilesystemT]: ''' Get a dictionary of labels to filesystems that are children of this filesystem. ''' - # Some filesystem classes allow hierarchical filesystem labels, - # permitting recursive snapshots to be taken on a filesystem and its - # children. An empty dictionary may be returned if this feature is not - # supported. - async def snapshots(self) -> Set[SnapshotKeyT]: ''' Get a set of keys for snapshots managed by this filesystem object. Snapshots taken on any children of the filesystem, @@ -82,4 +51,4 @@ class ZFSFilesystem(DispatcherMixin, FilesystemBase): a tag and a timestamp equal to that of the parent snapshot are deleted recursively. ''' -FilesystemRegistry.register(ZFSFilesystem) +FilesystemRegistry.register(ZFSFilesystem) \ No newline at end of file diff --git a/src/zasd/filesystems/zfs_old.py b/src/zasd/filesystems/zfs_old.txt similarity index 100% rename from src/zasd/filesystems/zfs_old.py rename to src/zasd/filesystems/zfs_old.txt diff --git a/src/zasd/repl.py b/src/zasd/repl.py index ccb1771..925bda8 100644 --- a/src/zasd/repl.py +++ b/src/zasd/repl.py @@ -1,38 +1,43 @@ ''' Interactive Python interpreter ''' -from typing import MutableMapping, Any, Callable, ContextManager, Dict, Optional +from typing import MutableMapping, Set, Any -import builtins -import asyncio +import sys import re -from ptpython.repl import PythonRepl -from prompt_toolkit.patch_stdout import patch_stdout as patch_stdout_context +import ptpython.repl +import zasd.entrypoint import zasd.config as config +from zasd.filesystem import FilesystemRegistry -@asyncio.coroutine async def repl(): - _globals = dict( - config = DictObject(config.reference(), 'config')) - - _locals = dict(a = 'b') - - # Create REPL. - _repl = PythonRepl( - get_globals=lambda: _globals, - get_locals=lambda: _locals, - vi_mode=True) - _repl.confirm_exit = False - _repl.highlight_matching_parenthesis = True - _repl.insert_blank_line_after_output = False + ''' Launch an asynchronous ptpython REPL ''' + + glo: MutableMapping = dict( + sys = sys, + config = DictObject(config.reference(), 'config'), + registry = FilesystemRegistry) + loc: MutableMapping = dict() + + def configure(ptrepl): + ptrepl.confirm_exit = False + ptrepl.highlight_matching_parenthesis = True + ptrepl.insert_blank_line_after_output = False + try: - with patch_stdout_context(): - await _repl.run_async() + await ptpython.repl.embed( + globals=glo, + locals=loc, + configure=configure, + return_asyncio_coroutine=True, + patch_stdout=True) except EOFError: - quit() + zasd.entrypoint.stop() + +class DictObject(): + ''' Class for wrapping dictionaries in objects ''' -class DictObject: def __init__(self, dictionary: MutableMapping[str, Any], name): DictObject._dictionary = dictionary DictObject._name = name @@ -50,24 +55,28 @@ class DictObject: raise AttributeError("name {} is not defined".format(name)) del self._dictionary[name] - def __dir__(self): + def __dir__(self) -> Set[str]: return DictObject._dictionary.keys() - def __repr__(self): - return DictObject.make_repr(DictObject._dictionary, DictObject._name) + def __repr__(self) -> str: + return DictObject.serialise(DictObject._dictionary, DictObject._name) @staticmethod - def make_repr(obj, name, level=0, path=[]): + def serialise(obj, name, level=0, path=None) -> str: + ''' Turn an object into a string recursively. ''' + if not isinstance(obj, dict): + # Output variables in 'object.name = value' format repstr = repr(obj) if re.search(r'^<.+>$', repstr): repstr = repstr[1:-1] - return '{}.{} = {}'.format(name, '.'.join(path), repstr) + return '{}.{} = {}'.format(name, '.'.join(path or []), repstr) else: + # Separate dictionary entries with newlines keys = sorted(obj.keys()) strs = list() for key in keys: - value = DictObject.make_repr( - obj[key], name, level + 1, path = path + [key]) + value = DictObject.serialise( + obj[key], name, level + 1, path = (path or []) + [key]) strs.append(value) - return '\n'.join(strs) + return '\n'.join(strs) \ No newline at end of file diff --git a/third_party/packages/six b/third_party/packages/six new file mode 160000 index 0000000..c0be881 --- /dev/null +++ b/third_party/packages/six @@ -0,0 +1 @@ +Subproject commit c0be8815d13df45b6ae471c4c436cce8c192245d diff --git a/vs.code-workspace b/vs.code-workspace index 0ac90b2..4771d01 100644 --- a/vs.code-workspace +++ b/vs.code-workspace @@ -8,8 +8,8 @@ "editor.rulers": [80], "terminal.integrated.env.osx": { "PYTHONPATH": "${workspaceFolder}/src:${workspaceFolder}/third_party/modules", - "MYPYPATH": "${workspaceFolder}/src:${workspaceFolder}/third_party/modules" +// "MYPYPATH": "${workspaceFolder}/src:${workspaceFolder}/third_party/modules" }, - "python.pythonPath": "/usr/local/bin/python" + //"python.pythonPath": "/usr/local/bin/python" } } \ No newline at end of file