Merge pull request #152 from powerapi-ng/fix/issue151/csv_input_only_postmortem

Fix/issue151/csv input only postmortem

Author: gfieni

powerapi/actor/actor.py

@@ -38,7 +38,7 @@
 
 from powerapi.exception import PowerAPIExceptionWithMessage
 from powerapi.message import PoisonPillMessage
-from powerapi.message import UnknowMessageTypeException
+from powerapi.message import UnknownMessageTypeException
 from powerapi.handler import HandlerException
 
 from .socket_interface import SocketInterface
@@ -228,7 +228,7 @@ def _initial_behaviour(self):
             try:
                 handler = self.state.get_corresponding_handler(msg)
                 handler.handle_message(msg)
-            except UnknowMessageTypeException:
+            except UnknownMessageTypeException:
                 self.logger.warning("UnknowMessageTypeException: " + str(msg))
             except HandlerException:
                 self.logger.warning("HandlerException")

powerapi/actor/state.py

@@ -27,7 +27,7 @@
 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-from powerapi.message import UnknowMessageTypeException
+from powerapi.message import UnknownMessageTypeException
 from powerapi.actor.supervisor import Supervisor
 
 
@@ -74,7 +74,7 @@ def get_corresponding_handler(self, msg):
         for (msg_type, handler) in self.handlers:
             if isinstance(msg, msg_type):
                 return handler
-        raise UnknowMessageTypeException()
+        raise UnknownMessageTypeException()
 
     def add_handler(self, message_type, handler):
         """

powerapi/cli/config_validator.py

@@ -68,6 +68,10 @@ def validate(config: Dict):
                 logging.error("no files parameter found for csv input")
                 return False
 
+            if input_config['type'] == 'csv' and config['stream']:
+                logging.error("stream mode cannot be used for csv input")
+                return False
+
             if 'model' not in input_config:
                 input_config['model'] = 'PowerReport'
             if 'name' not in input_config:

powerapi/dispatcher/dispatcher_actor.py

@@ -150,12 +150,11 @@ class DispatcherActor(Actor):
     """
 
     def __init__(self, name: str, formula_init_function: Callable, pushers: [], route_table: RouteTable,
-                 device_id: str, level_logger: Literal = logging.WARNING, timeout=None):
+                 level_logger: Literal = logging.WARNING, timeout=None):
         """
         :param str name: Actor name
         :param func formula_init_function: Function for creating Formula
         :param route_table: initialized route table of the DispatcherActor
-        :param device_id: name of the device the dispatcher handle
         :param int level_logger: Define the level of the logger
         :param bool timeout: Define the time in millisecond to wait for a
                              message before run timeout_handler
@@ -167,7 +166,6 @@ def __init__(self, name: str, formula_init_function: Callable, pushers: [], rout
 
         # (powerapi.DispatcherState): Actor state
         self.state = DispatcherState(self, self._create_factory(pushers), route_table)
-        self.device_id = device_id
 
     def setup(self):
         """

powerapi/dispatcher/simple/simple_dispatcher_handlers.py

@@ -28,7 +28,7 @@
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 from powerapi.actor import State
 from powerapi.handler import StartHandler, PoisonPillMessageHandler, Handler
-from powerapi.message import StartMessage, UnknowMessageTypeException
+from powerapi.message import StartMessage, UnknownMessageTypeException
 from powerapi.report import Report
 
 
@@ -75,4 +75,4 @@ def handle(self, msg: Report):
             self.state.actor.logger.debug('send ' + str(msg) + ' to ' + self.state.formula_name)
             self.state.formula.send_data(msg)
         else:
-            raise UnknowMessageTypeException()
+            raise UnknownMessageTypeException()

powerapi/formula/__init__.py

@@ -30,5 +30,5 @@
 from powerapi.formula.handlers import FormulaPoisonPillMessageHandler
 from powerapi.formula.abstract_cpu_dram_formula import AbstractCpuDramFormula
 from powerapi.formula.dummy.dummy_formula_actor import DummyFormulaActor
-from powerapi.formula.formula_actor import FormulaActor, FormulaState, DomainValues
+from powerapi.formula.formula_actor import FormulaActor, FormulaState
 from powerapi.formula.simple.simple_formula_actor import SimpleFormulaActor

powerapi/formula/formula_actor.py

@@ -30,7 +30,7 @@
 import logging
 import re
 
-from typing import Dict, Tuple
+from typing import Dict
 
 from powerapi.actor import Actor, State
 from powerapi.pusher import PusherActor
@@ -50,16 +50,6 @@ def __init__(self, actor, pushers, metadata):
         self.metadata = metadata
 
 
-class DomainValues:
-    """
-    values that describe the device that the formula compute the power consumption for
-    """
-
-    def __init__(self, device_id: str, formula_id: Tuple):
-        self.device_id = device_id
-        self.sensor = formula_id[0]
-
-
 class FormulaActor(Actor):
     """
     Abstract actor class used to implement formula actor that compute power consumption of a device from Reports

powerapi/handler/handler.py

@@ -28,7 +28,7 @@
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 from powerapi.exception import PowerAPIException
-from powerapi.message import Message, UnknowMessageTypeException
+from powerapi.message import Message, UnknownMessageTypeException
 
 
 class HandlerException(PowerAPIException):
@@ -83,7 +83,7 @@ def delegate_message_handling(self, msg: Message):
         try:
             handler = self.state.get_corresponding_handler(msg)
             handler.handle_message(msg)
-        except UnknowMessageTypeException:
+        except UnknownMessageTypeException:
             self.state.actor.logger.warning("UnknowMessageTypeException: " + str(msg))
         except HandlerException:
             self.state.actor.logger.warning("HandlerException")

powerapi/handler/poison_pill_message_handler.py

@@ -27,7 +27,7 @@
 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-from powerapi.message import UnknowMessageTypeException, PoisonPillMessage
+from powerapi.message import UnknownMessageTypeException, PoisonPillMessage
 from .handler import Handler
 
 
@@ -68,7 +68,7 @@ def handle(self, msg):
         :param Object msg: the message received by the actor
         """
         if not isinstance(msg, PoisonPillMessage):
-            raise UnknowMessageTypeException(type(msg))
+            raise UnknownMessageTypeException(type(msg))
 
         if msg.is_soft:
             self._empty_mail_box()

powerapi/message.py

@@ -36,7 +36,7 @@
     from powerapi.database import BaseDB
     from powerapi.filter import Filter
     from powerapi.dispatcher import RouteTable
-    from powerapi.formula import FormulaActor, FormulaState, DomainValues
+    from powerapi.formula import FormulaActor, FormulaState
     from powerapi.report_modifier import ReportModifier
 
 
@@ -172,7 +172,7 @@ def __eq__(self, other):
         return False
 
 
-class UnknowMessageTypeException(PowerAPIException):
+class UnknownMessageTypeException(PowerAPIException):
     """
     Exception happen when we don't know the message type
     """

powerapi/puller/simple/simple_puller_handlers.py

@@ -29,7 +29,7 @@
 
 from powerapi.actor import State
 from powerapi.handler import Handler, StartHandler
-from powerapi.message import Message, SimplePullerSendReportsMessage, UnknowMessageTypeException
+from powerapi.message import Message, SimplePullerSendReportsMessage, UnknownMessageTypeException
 from powerapi.puller.handlers import PullerInitializationException
 
 
@@ -72,4 +72,4 @@ def handle(self, msg: Message):
                 sent += 1
             self.state.actor.logger.debug('sent reports: ' + str(sent))
         else:
-            raise UnknowMessageTypeException()
+            raise UnknownMessageTypeException()

tests/__init__.py

@@ -0,0 +1,28 @@
+# Copyright (c) 2023, INRIA
+# Copyright (c) 2023, University of Lille
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# * Redistributions of source code must retain the above copyright notice, this
+#   list of conditions and the following disclaimer.
+#
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+#
+# * Neither the name of the copyright holder nor the names of its
+#   contributors may be used to endorse or promote products derived from
+#   this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

tests/acceptation/conftest.py

@@ -0,0 +1,63 @@
+# Copyright (c) 2023, INRIA
+# Copyright (c) 2023, University of Lille
+# All rights reserved.
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+
+# * Redistributions of source code must retain the above copyright notice, this
+#   list of conditions and the following disclaimer.
+
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+
+# * Neither the name of the copyright holder nor the names of its
+#   contributors may be used to endorse or promote products derived from
+#   this software without specific prior written permission.
+
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+from multiprocessing import active_children
+
+import pytest
+
+from tests.utils.db.mongo import MONGO_URI, MONGO_DATABASE_NAME, MONGO_OUTPUT_COLLECTION_NAME
+
+
+@pytest.fixture
+def shutdown_system():
+    """
+    Shutdown the actor system, i.e., all actors are killed
+    """
+    yield None
+    active = active_children()
+    for child in active:
+        child.kill()
+
+
+@pytest.fixture()
+def socket_info_config():
+    """
+    Return a configuration with socket as input and mongo as output
+    """
+    return {'verbose': True,
+            'stream': False,
+            'input': {'test_puller': {'type': 'socket',
+                                      'port': 0,
+                                      'model': 'HWPCReport'}},
+            'output': {'test_pusher': {'type': 'mongodb',
+                                       'uri': MONGO_URI,
+                                       'db': MONGO_DATABASE_NAME,
+                                       'model': 'PowerReport',
+                                       'max_buffer_size': 0,
+                                       'collection': MONGO_OUTPUT_COLLECTION_NAME}},
+            }

tests/acceptation/test_simple_architecture.py

@@ -47,9 +47,7 @@
   - each HWPCReport in the intput database was converted in one PowerReport per
     socket in the output database
 """
-# pylint: disable=redefined-outer-name
-# pylint: disable=unused-argument
-# pylint: disable=unused-import
+# pylint: disable=redefined-outer-name,unused-argument,unused-import
 import time
 from datetime import datetime
 
@@ -59,8 +57,7 @@
 from powerapi.actor import Supervisor
 from powerapi.formula.dummy import DummyFormulaActor
 
-# noinspection PyUnresolvedReferences
-from tests.utils.acceptation import launch_simple_architecture, socket_info_config, BASIC_CONFIG, SOCKET_DEPTH_LEVEL, \
+from tests.utils.acceptation import launch_simple_architecture, BASIC_CONFIG, SOCKET_DEPTH_LEVEL, \
     INFLUX_OUTPUT_CONFIG, CSV_INPUT_OUTPUT_CONFIG
 from tests.utils.report.hwpc import extract_rapl_reports_with_2_sockets
 # noinspection PyUnresolvedReferences
@@ -72,9 +69,6 @@
 from tests.utils.db.csv import ROOT_PATH, OUTPUT_PATH, files
 from tests.utils.db.socket import ClientThread, ClientThreadDelay
 
-# noinspection PyUnresolvedReferences
-from tests.utils.unit import shutdown_system
-
 
 ##################
 # MONGO to MONGO #

tests/acceptation/test_simple_architecture_with_libvirt_mapper.py

@@ -48,9 +48,7 @@
     socket in the output database
   - only target name LIBVIRT_TARGET_NAME1 was converted into UUID_1
 """
-# pylint: disable=redefined-outer-name
-# pylint: disable=unused-argument
-# pylint: disable=unused-import
+# pylint: disable=redefined-outer-name,disable=unused-argument,disable=unused-import
 import time
 from datetime import datetime
 from mock import patch
@@ -66,8 +64,6 @@
     MONGO_DATABASE_NAME, mongo_database
 from tests.utils.report.hwpc import extract_all_events_reports_with_vm_name
 from tests.utils.libvirt import MockedLibvirt, LIBVIRT_TARGET_NAME1, UUID_1
-# noinspection PyUnresolvedReferences
-from tests.utils.unit import shutdown_system
 
 
 @pytest.fixture

tests/integration/cli/test_generate_pusher.py

@@ -36,8 +36,6 @@
 from powerapi.utils import timestamp_to_datetime
 # noinspection PyUnresolvedReferences
 from tests.utils.db.influx import INFLUX_URI, INFLUX_PORT, INFLUX_DBNAME, influx_database
-# noinspection PyUnresolvedReferences
-from tests.utils.unit import shutdown_system
 
 SENSOR_NAME = 'sensor_test'
 TARGET_NAME = 'system'

tests/utils/unit.py renamed to tests/integration/conftest.py

@@ -1,21 +1,21 @@
-# Copyright (c) 2022, INRIA
-# Copyright (c) 2022, University of Lille
+# Copyright (c) 2023, INRIA
+# Copyright (c) 2023, University of Lille
 # All rights reserved.
-#
+
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are met:
-#
+
 # * Redistributions of source code must retain the above copyright notice, this
 #   list of conditions and the following disclaimer.
-#
+
 # * Redistributions in binary form must reproduce the above copyright notice,
 #   this list of conditions and the following disclaimer in the documentation
 #   and/or other materials provided with the distribution.
-#
+
 # * Neither the name of the copyright holder nor the names of its
 #   contributors may be used to endorse or promote products derived from
 #   this software without specific prior written permission.
-#
+
 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
@@ -26,10 +26,10 @@
 # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-from multiprocessing import active_children
-
 import pytest
 
+from multiprocessing import active_children
+
 
 @pytest.fixture
 def shutdown_system():

tests/integration/test_dispatcher.py

@@ -38,8 +38,6 @@
 from tests.utils.dummy_actor import DummyActor
 from tests.utils.report.hwpc import gen_HWPCReports
 from tests.unit.actor.abstract_test_actor import recv_from_pipe
-# noinspection PyUnresolvedReferences
-from tests.utils.unit import shutdown_system
 
 PUSHER_NAME_POWER_REPORT = 'fake_pusher_power'
 REPORT_TYPE_TO_BE_SENT = PowerReport
@@ -92,8 +90,7 @@ def started_dispatcher(started_fake_pusher_power_report, route_table):
                                                                                           pushers=pushers, socket=0,
                                                                                           core=0),
                             route_table=route_table,
-                            pushers={PUSHER_NAME_POWER_REPORT: started_fake_pusher_power_report},
-                            device_id='test_device')
+                            pushers={PUSHER_NAME_POWER_REPORT: started_fake_pusher_power_report})
     actor.start()
     actor.connect_data()
     actor.connect_control()