Skip to content

Commit 7a93c5f

Browse files
JSON logging facility (#56)
* Add machine-readable output (JSON logger) * Improve default human / CLI logger output --------- Co-authored-by: Johann Wagner <[email protected]>
1 parent 4a1a17e commit 7a93c5f

20 files changed

+491
-183
lines changed

cosmo/__main__.py

Lines changed: 15 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,29 +2,39 @@
22
import os
33
import sys
44
import pathlib
5-
import warnings
65

76
import yaml
87
import argparse
98

109
from cosmo.clients.netbox import NetboxClient
11-
from cosmo.log import info
10+
from cosmo.log import info, logger, JsonLoggingStrategy, error, HumanReadableLoggingStrategy
1211
from cosmo.serializer import RouterSerializer, SwitchSerializer
1312
from cosmo.common import AbstractRecoverableError
1413

1514

1615

1716
def main() -> int:
17+
sys.excepthook = logger.exceptionHook
18+
1819
parser = argparse.ArgumentParser(
1920
description="Automagically generate filter lists and BGP sessions for WAN-Core network"
2021
)
2122
parser.add_argument('--limit', default=[], metavar="STRING", action="append",
2223
help='List of hosts to generate configurations')
2324
parser.add_argument('--config', '-c', default='cosmo.yml', metavar="CFGFILE",
2425
help='Path of the yaml config file to use')
26+
parser.add_argument('--json', '-j', action="store_true",
27+
help='Toggle machine readable output on')
2528

2629
args = parser.parse_args()
2730

31+
if args.json:
32+
logger.setLoggingStrategy(JsonLoggingStrategy())
33+
else:
34+
logger.setLoggingStrategy(HumanReadableLoggingStrategy(
35+
netbox_instance_url=str(os.environ.get("NETBOX_URL")) # isn't validated so it's fine
36+
))
37+
2838
if len(args.limit) > 1:
2939
allowed_hosts = args.limit
3040
elif len(args.limit) == 1 and args.limit[0] != "ci":
@@ -68,7 +78,7 @@ def noop(*args, **kwargs):
6878
if allowed_hosts and device['name'] not in allowed_hosts and device_fqdn not in allowed_hosts:
6979
continue
7080

71-
info(f"Generating {device_fqdn}")
81+
info(f"generating...", device_fqdn)
7282

7383
content = None
7484
try:
@@ -79,7 +89,7 @@ def noop(*args, **kwargs):
7989
switch_serializer = SwitchSerializer(device)
8090
content = switch_serializer.serialize()
8191
except AbstractRecoverableError as e:
82-
warnings.warn(f"{device['name']} serialization error \"{e}\", skipping ...")
92+
error(f"{device['name']} serialization error \"{e}\", skipping ...", device_fqdn)
8393
continue
8494

8595
match cosmo_configuration['output_format']:
@@ -103,8 +113,8 @@ def noop(*args, **kwargs):
103113
json.dump(content, json_file, indent=4)
104114
case other:
105115
raise Exception(f"unsupported output format {other}")
106-
return 1
107116

117+
logger.flush()
108118
return 0
109119

110120

cosmo/abstractroutervisitor.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
from abc import ABC
22

3-
from cosmo.types import CosmoLoopbackType
3+
from cosmo.netbox_types import CosmoLoopbackType
44
from cosmo.visitors import AbstractNoopNetboxTypesVisitor
55

66

cosmo/clients/netbox_v4.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -366,6 +366,7 @@ def _fetch_data(self, kwargs):
366366
}
367367
tags {
368368
__typename
369+
id
369370
name
370371
slug
371372
}

cosmo/common.py

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
import abc
22

3+
APP_NAME = "cosmo"
4+
35
class AbstractRecoverableError(Exception, abc.ABC):
46
pass
57

@@ -23,6 +25,8 @@ class L2VPNSerializationError(AbstractRecoverableError):
2325
# the visitors will export.
2426
CosmoOutputType = dict[str, str|dict[str, "CosmoOutputType"]|list["CosmoOutputType"]]
2527

28+
JsonOutputType = CosmoOutputType
29+
2630
# next() can raise StopIteration, so that's why I use this function
2731
def head(l):
2832
return None if not l else l[0]

cosmo/cperoutervisitor.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
from functools import singledispatchmethod
22
from ipaddress import IPv6Network, IPv4Network
33

4-
from cosmo.types import IPAddressType, DeviceType
4+
from cosmo.netbox_types import IPAddressType, DeviceType
55
from cosmo.visitors import AbstractNoopNetboxTypesVisitor
66

77

cosmo/l2vpnhelpertypes.py

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,20 @@
11
import ipaddress
2-
import warnings
32
from abc import abstractmethod, ABCMeta
43
from functools import singledispatchmethod
54
from typing import NoReturn
65

76
from cosmo.abstractroutervisitor import AbstractRouterExporterVisitor
87
from cosmo.common import head, CosmoOutputType, L2VPNSerializationError
9-
from cosmo.types import InterfaceType, VLANType, AbstractNetboxType, DeviceType, L2VPNType, CosmoLoopbackType, \
8+
from cosmo.log import warn
9+
from cosmo.netbox_types import InterfaceType, VLANType, AbstractNetboxType, DeviceType, L2VPNType, CosmoLoopbackType, \
1010
L2VPNTerminationType
1111

1212

1313
# FIXME simplify this!
1414
class AbstractEncapCapability(metaclass=ABCMeta):
1515
@singledispatchmethod
1616
def accept(self, o):
17-
warnings.warn(f"cannot find suitable encapsulation for type {type(o)}")
17+
warn(f"cannot find suitable encapsulation.", o)
1818

1919

2020
class EthernetCccEncapCapability(AbstractEncapCapability, metaclass=ABCMeta):
@@ -113,11 +113,11 @@ def getChosenEncapType(self, o: AbstractNetboxType) -> str | None:
113113
return chosen_encap
114114

115115
def processInterfaceTypeTermination(self, o: InterfaceType) -> dict | None:
116-
warnings.warn(f"{self.getNetboxTypeName().upper()} L2VPN does not support {type(o)} terminations.")
116+
warn(f"{self.getNetboxTypeName().upper()} L2VPN does not support {type(o)} terminations.", o)
117117
return None
118118

119119
def processVLANTypeTermination(self, o: VLANType) -> dict | None:
120-
warnings.warn(f"{self.getNetboxTypeName().upper()} L2VPN does not support {type(o)} terminations.")
120+
warn(f"{self.getNetboxTypeName().upper()} L2VPN does not support {type(o)} terminations.", o)
121121
return None
122122

123123
def spitInterfaceEncapFor(self, o: VLANType|InterfaceType):

cosmo/log.py

Lines changed: 187 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,187 @@
1-
def info(string: str) -> None:
2-
print("[INFO] " + string)
1+
import json
2+
import sys
3+
from abc import abstractmethod, ABCMeta
4+
from typing import Self
5+
6+
from termcolor import colored
7+
8+
from cosmo.common import AbstractRecoverableError, JsonOutputType
9+
from cosmo.netbox_types import AbstractNetboxType
10+
11+
12+
class AbstractLogLevel(metaclass=ABCMeta):
13+
name: str = "abstract_log_level"
14+
def __str__(self):
15+
return self.name.upper()
16+
17+
class InfoLogLevel(AbstractLogLevel):
18+
name = "info"
19+
class WarningLogLevel(AbstractLogLevel):
20+
name = "warning"
21+
class ErrorLogLevel(AbstractLogLevel):
22+
name = "error"
23+
24+
25+
O = object | AbstractNetboxType | None # object-being-logged-on type
26+
M = tuple[AbstractLogLevel, str, O] # message type
27+
28+
class AbstractLoggingStrategy(metaclass=ABCMeta):
29+
@abstractmethod
30+
def flush(self): # this is for async and sync logging
31+
pass
32+
@abstractmethod
33+
def info(self, message: str, on: O):
34+
pass
35+
@abstractmethod
36+
def warn(self, message: str, on: O):
37+
pass
38+
@abstractmethod
39+
def error(self, message: str, on: O):
40+
pass
41+
@abstractmethod
42+
def exceptionHook(self, exception: type[BaseException], value: BaseException, traceback):
43+
pass
44+
45+
46+
class JsonLoggingStrategy(AbstractLoggingStrategy):
47+
info_queue: list[M] = []
48+
warning_queue: list[M] = []
49+
error_queue: list[M] = []
50+
51+
@staticmethod
52+
def _messageToJSON(m: M):
53+
log_level, message, obj = m
54+
return {
55+
"level": log_level.name,
56+
"message": message,
57+
"object": (obj.getMetaInfo().toJSON() if isinstance(obj, AbstractNetboxType) else {
58+
"type": type(obj).__name__, "value": str(obj)
59+
}),
60+
}
61+
62+
def info(self, message: str, on: O):
63+
self.info_queue.append((InfoLogLevel(), message, on))
64+
65+
def warn(self, message: str, on: O):
66+
self.warning_queue.append((WarningLogLevel(), message, on))
67+
68+
def error(self, message: str, on: O):
69+
self.error_queue.append((ErrorLogLevel(), message, on))
70+
71+
def flush(self):
72+
# JSON-RPC like
73+
res = {}
74+
if len(self.warning_queue) + len(self.error_queue) == 0:
75+
res = {
76+
"result": list(map(self._messageToJSON, self.info_queue)),
77+
}
78+
else:
79+
res = {
80+
"error": list(map(self._messageToJSON, self.error_queue)),
81+
"warning": list(map(self._messageToJSON, self.warning_queue)),
82+
}
83+
print(json.dumps(res))
84+
85+
def exceptionHook(self, exception: type[BaseException], value: BaseException, traceback):
86+
if isinstance(exception, AbstractRecoverableError):
87+
self.warn(str(value), None)
88+
else:
89+
self.error(str(value), None)
90+
91+
92+
class HumanReadableLoggingStrategy(AbstractLoggingStrategy):
93+
def __init__(self, *args, netbox_instance_url: str, **kwargs):
94+
super().__init__(*args, **kwargs)
95+
self.nb_instance_url = netbox_instance_url
96+
97+
def formatMessage(self, m: M) -> str:
98+
log_level, message, obj = m
99+
match log_level:
100+
case InfoLogLevel():
101+
color = "blue"
102+
case WarningLogLevel():
103+
color = "yellow"
104+
case ErrorLogLevel():
105+
color = "red"
106+
case _:
107+
color = "white"
108+
log_level_colored = colored(log_level, color)
109+
default_log = f"[{log_level_colored}] {message}"
110+
match obj:
111+
case AbstractNetboxType():
112+
meta_info = obj.getMetaInfo()
113+
full_url = meta_info.getFullObjectURL(self.nb_instance_url)
114+
return (
115+
f"[{log_level_colored}]"
116+
f" [{meta_info.device_display_name.lower()}]"
117+
f" [{meta_info.display_name}] "
118+
f"{message}\n" +
119+
colored(f"🌐 {full_url}", "light_blue")
120+
)
121+
case None:
122+
return default_log
123+
case str()|object():
124+
return f"[{log_level_colored}] [{obj}] {message}"
125+
case _:
126+
return default_log
127+
128+
def info(self, message: str, on: O):
129+
print(self.formatMessage((InfoLogLevel(), message, on)))
130+
131+
def warn(self, message: str, on: O):
132+
print(self.formatMessage((WarningLogLevel(), message, on)))
133+
134+
def error(self, message: str, on: O):
135+
print(self.formatMessage((ErrorLogLevel(), message, on)))
136+
137+
def flush(self):
138+
pass
139+
140+
def exceptionHook(self, exception: type[BaseException], value: BaseException, traceback):
141+
sys.__excepthook__(exception, value, traceback)
142+
143+
144+
class CosmoLogger:
145+
strategy: AbstractLoggingStrategy
146+
147+
def setLoggingStrategy(self, strategy: AbstractLoggingStrategy) -> Self:
148+
self.strategy = strategy
149+
return self
150+
151+
def flush(self) -> Self:
152+
self.strategy.flush()
153+
return self
154+
155+
def getLoggingStrategy(self) -> AbstractLoggingStrategy:
156+
return self.strategy
157+
158+
def info(self, message: str, on: O):
159+
self.strategy.info(message, on)
160+
161+
def warn(self, message: str, on: O):
162+
self.strategy.warn(message, on)
163+
164+
def error(self, message: str, on: O):
165+
self.strategy.error(message, on)
166+
167+
def processHandledException(self, exception: BaseException): # for try/catch blocks to use
168+
self.exceptionHook(type(exception), exception, None, recovered=True)
169+
170+
def exceptionHook(self, exception: type[BaseException], value: BaseException, traceback, recovered=False):
171+
self.strategy.exceptionHook(exception, value, traceback)
172+
if not recovered:
173+
# not recoverable because uncaught (we've been called from sys.excepthook,
174+
# since recovered is False by default). we're stopping the interpreter NOW.
175+
self.flush()
176+
177+
178+
def info(message: str, on: O = None) -> None:
179+
logger.info(message, on)
180+
181+
def warn(message: str, on: O) -> None:
182+
logger.warn(message, on)
183+
184+
def error(message: str, on:O) -> None:
185+
logger.error(message, on)
186+
187+
logger = CosmoLogger()

cosmo/manufacturers.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
from typing import NoReturn
44

55
from cosmo.common import DeviceSerializationError
6-
from cosmo.types import DeviceType, InterfaceType
6+
from cosmo.netbox_types import DeviceType, InterfaceType
77

88

99
class AbstractManufacturer(ABC):

0 commit comments

Comments
 (0)