Add tests for utils.py, ComponentPayload.py

griko · griko · commit 710e570ce663 · 2023-01-14T00:52:11.000+02:00
diff --git a/pipeline.yaml b/pipeline.yaml
@@ -32,6 +32,7 @@ preprocessing:
     output_dir: 'pyannote_vad_preprocessed'
     add_segment_metadata: true
     performance_measurement: true
+    keep_only_first_segment: true
     onset: 0.85             # onset activation threshold
     offset: 0.8             # offset activation threshold
     min_duration_on: 0.1    # remove speech regions shorter than that many seconds
diff --git a/pyproject.toml b/pyproject.toml
@@ -0,0 +1,3 @@
+[build-system]
+requires = ["setuptools>=42.0", "wheel"]
+build-backend = "setuptools.build_meta"
diff --git a/setup.cfg b/setup.cfg
@@ -1,2 +1,34 @@
 [flake8]
 max-line-length=120
+
+[metadata]
+name = vanpy
+description = Voice ANalysis framework
+author = Gregory Koushnir
+license = Apache License v2.0
+license_file = LICENSE
+platforms = unix, linux, osx, cygwin, win32
+classifiers =
+    Programming Language :: Python :: 3
+    Programming Language :: Python :: 3 :: Only
+    Programming Language :: Python :: 3.6
+    Programming Language :: Python :: 3.7
+    Programming Language :: Python :: 3.8
+    Programming Language :: Python :: 3.9
+
+[options]
+packages =
+    vanpy
+python_requires = >=3.6
+package_dir =
+    =src
+zip_safe = no
+
+[options.extras_require]
+testing =
+    pytest>=6.0
+    pytest-cov>=2.0
+    mypy>=0.910
+    flake8>=3.9
+    tox>=3.24
+
diff --git a/setup.py b/setup.py
@@ -10,7 +10,7 @@
     description='Voice ANalysis framework',
     author='Gregory Koushnir',
     author_email='koushgre@post.bgu.ac.il',
-    packages=find_packages(include=['vanpy.core', 'vanpy.core.*', 'vanpy.utils', 'vanpy.utils.*']),
+    # packages=find_packages(include=['vanpy.core', 'vanpy.core.*', 'vanpy.utils', 'vanpy.utils.*']),
     install_requires=[
         'gdown>=4.4.0',
         'huggingface-hub>=0.5.1',
diff --git a/src/vanpy/__init__.py b/src/vanpy/__init__.py
@@ -1 +1 @@
-__version__ = '0.71.31'
+__version__ = '0.71.33'
diff --git a/src/vanpy/core/BasePipline.py b/src/vanpy/core/BasePipline.py
@@ -6,7 +6,7 @@
 from yaml import YAMLObject
 
 from src.vanpy.core.ComponentPayload import ComponentPayload
-from src.vanpy.core.PiplineComponent import PipelineComponent
+from src.vanpy.core.PipelineComponent import PipelineComponent
 
 
 @dataclass
diff --git a/src/vanpy/core/CombinedPipeline.py b/src/vanpy/core/CombinedPipeline.py
@@ -5,7 +5,7 @@
 from typing import List
 
 from src.vanpy.core.BasePipline import BasePipeline
-from src.vanpy.core.PiplineComponent import ComponentPayload
+from src.vanpy.core.PipelineComponent import ComponentPayload
 from yaml import YAMLObject
 
 
diff --git a/src/vanpy/core/ComponentPayload.py b/src/vanpy/core/ComponentPayload.py
@@ -69,7 +69,7 @@ def get_columns(self, all_paths_columns=False, meta_columns=False):
             columns.extend(self.metadata['meta_columns'])
         return columns
 
-    def get_declared_columns(self, ext_columns: List[str], all_paths_columns=False, meta_columns=False):
+    def get_declared_columns(self, ext_columns: List[str], all_paths_columns=False, meta_columns=False) -> pd.DataFrame:
         """
         Returns a payload's dataframe containing the specified columns.
 
@@ -85,7 +85,7 @@ def get_declared_columns(self, ext_columns: List[str], all_paths_columns=False,
         columns = self.get_columns(all_paths_columns, meta_columns)
         for cols in ext_columns:
             columns.extend(self.metadata[cols])
-        columns = list(set(columns) & set(self.df.columns))
+        # columns = list(set(columns) & set(self.df.columns))
         return self.df[columns]
 
     def get_features_df(self, all_paths_columns=False, meta_columns=False):
diff --git a/src/vanpy/core/PipelineComponent.py b/src/vanpy/core/PipelineComponent.py
@@ -126,5 +126,13 @@ def save_component_payload(self, input_payload: ComponentPayload, intermediate=F
             self.get_logger().info(f'Saved payload in {self.config["intermediate_payload_path"]}')
 
     def save_intermediate_payload(self, i: int, input_payload: ComponentPayload):
+        """
+        Save intermediate payload based on the save_payload_periodicity configuration.
+
+        :param i: current iteration count
+        :type i: int
+        :param input_payload: the payload to be saved
+        :type input_payload: ComponentPayload
+        """
         if 'save_payload_periodicity' in self.config and i % self.config['save_payload_periodicity'] == 0 and i > 0:
             self.save_component_payload(input_payload, intermediate=True)
diff --git a/src/vanpy/core/feature_extraction_components/LibrosaFeaturesExtractor.py b/src/vanpy/core/feature_extraction_components/LibrosaFeaturesExtractor.py
@@ -7,7 +7,7 @@
 import pandas as pd
 
 from src.vanpy.core.ComponentPayload import ComponentPayload
-from src.vanpy.core.PiplineComponent import PipelineComponent
+from src.vanpy.core.PipelineComponent import PipelineComponent
 from typing import List
 
 
diff --git a/src/vanpy/core/feature_extraction_components/PyannoteEmbedding.py b/src/vanpy/core/feature_extraction_components/PyannoteEmbedding.py
@@ -8,7 +8,7 @@
 import torch
 
 from src.vanpy.core.ComponentPayload import ComponentPayload
-from src.vanpy.core.PiplineComponent import PipelineComponent
+from src.vanpy.core.PipelineComponent import PipelineComponent
 from src.vanpy.utils.utils import get_null_wav_path
 
 
diff --git a/src/vanpy/core/feature_extraction_components/SpeechBrainEmbedding.py b/src/vanpy/core/feature_extraction_components/SpeechBrainEmbedding.py
@@ -6,7 +6,7 @@
 import torch
 import pandas as pd
 from src.vanpy.core.ComponentPayload import ComponentPayload
-from src.vanpy.core.PiplineComponent import PipelineComponent
+from src.vanpy.core.PipelineComponent import PipelineComponent
 from src.vanpy.utils.utils import get_null_wav_path
 
 
diff --git a/src/vanpy/core/preprocess_components/FilelistDataFrameCreator.py b/src/vanpy/core/preprocess_components/FilelistDataFrameCreator.py
@@ -1,7 +1,7 @@
 import pandas as pd
 
 from src.vanpy.core.ComponentPayload import ComponentPayload
-from src.vanpy.core.PiplineComponent import PipelineComponent
+from src.vanpy.core.PipelineComponent import PipelineComponent
 from src.vanpy.utils.utils import get_audio_files_paths
 from yaml import YAMLObject
 import pickle
diff --git a/src/vanpy/core/preprocess_components/SegmenterComponent.py b/src/vanpy/core/preprocess_components/SegmenterComponent.py
@@ -2,7 +2,7 @@
 
 from yaml import YAMLObject
 
-from src.vanpy.core.PiplineComponent import PipelineComponent
+from src.vanpy.core.PipelineComponent import PipelineComponent
 from src.vanpy.utils.utils import get_audio_files_paths
 import pandas as pd
 
diff --git a/src/vanpy/core/segment_classification_components/CVAgeClassifier.py b/src/vanpy/core/segment_classification_components/CVAgeClassifier.py
@@ -2,7 +2,7 @@
 from yaml import YAMLObject
 
 from src.vanpy.core.ComponentPayload import ComponentPayload
-from src.vanpy.core.PiplineComponent import PipelineComponent
+from src.vanpy.core.PipelineComponent import PipelineComponent
 from src.vanpy.utils.utils import cached_download
 
 
diff --git a/src/vanpy/core/segment_classification_components/CVGenderClassifier.py b/src/vanpy/core/segment_classification_components/CVGenderClassifier.py
@@ -2,7 +2,7 @@
 from yaml import YAMLObject
 
 from src.vanpy.core.ComponentPayload import ComponentPayload
-from src.vanpy.core.PiplineComponent import PipelineComponent
+from src.vanpy.core.PipelineComponent import PipelineComponent
 from src.vanpy.utils.utils import cached_download
 
 
diff --git a/src/vanpy/core/segment_classification_components/IEMOCAPEmotionClassifier.py b/src/vanpy/core/segment_classification_components/IEMOCAPEmotionClassifier.py
@@ -2,7 +2,7 @@
 from yaml import YAMLObject
 
 from src.vanpy.core.ComponentPayload import ComponentPayload
-from src.vanpy.core.PiplineComponent import PipelineComponent
+from src.vanpy.core.PipelineComponent import PipelineComponent
 from speechbrain.pretrained.interfaces import foreign_class
 
 
diff --git a/src/vanpy/core/segment_classification_components/Wav2Vec2STT.py b/src/vanpy/core/segment_classification_components/Wav2Vec2STT.py
@@ -5,7 +5,7 @@
 from yaml import YAMLObject
 from transformers import Wav2Vec2ForCTC, Wav2Vec2Tokenizer
 from src.vanpy.core.ComponentPayload import ComponentPayload
-from src.vanpy.core.PiplineComponent import PipelineComponent
+from src.vanpy.core.PipelineComponent import PipelineComponent
 
 
 class Wav2Vec2STT(PipelineComponent):
diff --git a/src/vanpy/utils/utils.py b/src/vanpy/utils/utils.py
@@ -1,6 +1,6 @@
 import os
 import subprocess
-from typing import List, Tuple
+from typing import List, Tuple, Dict
 import gdown
 import logging
 import yaml
@@ -9,12 +9,25 @@
 
 
 def create_dirs_if_not_exist(*args: str) -> None:
+    """
+    Create directories if they do not exist.
+    :param args: directory paths
+    """
     for arg in args:
         os.makedirs(arg, exist_ok=True)
         # logger.info(f'Created dir {arg}')
 
 
 def cut_segment(input_path: str, output_dir: str, segment: Tuple[float, float], segment_id: int, separator: str) -> str:
+    """
+    Cut a segment of audio from a given file.
+    :param input_path: path to audio file
+    :param output_dir: directory where the segmented audio file should be stored
+    :param segment: start and end time of the segment in seconds
+    :param segment_id: id of the segment
+    :param separator: separator to use in the output file name
+    :return: path of the segmented audio file
+    """
     create_dirs_if_not_exist(output_dir)
     start, stop = segment
     f = ''.join(str(input_path).split("/")[-1].split(".")[:-1])
@@ -26,21 +39,40 @@ def cut_segment(input_path: str, output_dir: str, segment: Tuple[float, float],
 
 
 def get_audio_files_paths(folder: str, extension: str = '') -> List[str]:
+    """
+    Get a list of audio files in a given folder.
+    :param folder: path of the folder
+    :param extension: extension of the audio files
+    :return: list of audio file paths
+    """
     folder_files = os.listdir(folder)
     logger.info(f'Finished listdir on {folder}')
     return [f'{folder}/{f}' for f in folder_files if f.endswith(extension) and os.path.isfile(f'{folder}/{f}')]
 
 
-def cached_download(url, path):
+def cached_download(url, path) -> str:
+    """
+    Download a file from a given URL and save it to a specified path. If the file already exists, skip the download.
+    :param url: URL of the file
+    :param path: path to save the file
+    :return: path of the downloaded file
+    """
     if os.path.exists(path):
         pass
     else:
-        create_dirs_if_not_exist('/'.join(path.split('/')[:-1]))
+        create_dirs_if_not_exist(os.sep.join(path.split(os.sep)[:-1]))
         gdown.download(url, path, quiet=True)
     return path
 
 
-def yaml_placeholder_replacement(full, val=None, initial=True):
+def yaml_placeholder_replacement(full, val=None, initial=True) -> yaml.YAMLObject:
+    """
+    Replace placeholders in a YAML file with their corresponding values.
+    :param full: full YAML file
+    :param val: current value being processed
+    :param initial: flag to indicate if this is the initial call of the function
+    :return: YAML file with placeholders replaced
+    """
     val = val or full if initial else val
     if isinstance(val, dict):
         for k, v in val.items():
@@ -55,14 +87,23 @@ def yaml_placeholder_replacement(full, val=None, initial=True):
     return val
 
 
-def load_config(config_yaml_path:str = 'pipeline.yaml'):
+def load_config(config_yaml_path:str = 'pipeline.yaml') -> Dict:
+    """
+    Load a YAML configuration file and replace any placeholders with their corresponding values.
+    :param config_yaml_path: path of the configuration file
+    :return: configuration as a dictionary
+    """
     with open(config_yaml_path, 'r') as f:
         config = yaml.load(f, Loader=yaml.FullLoader)
         config = yaml_placeholder_replacement(config)
     return config
 
 
-def get_null_wav_path():
+def get_null_wav_path() -> str:
+    """
+    Get the path of a null wav file.
+    :return: path of the null wav file
+    """
     path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "empty.wav")
     if not os.path.isfile(path):
         gdown.download('https://drive.google.com/uc?export=download&confirm=9iBg&id=1URDocYaa0tKe3KLiFJd5ct7tsczA3mX4', path, quiet=True)
diff --git a/tests/test_ComponentPayload.py b/tests/test_ComponentPayload.py
@@ -0,0 +1,33 @@
+import unittest
+import pandas as pd
+
+from src.vanpy.core.ComponentPayload import ComponentPayload
+
+
+class TestComponentPayload(unittest.TestCase):
+    def setUp(self):
+        self.input_path = 'data.csv'
+        self.metadata = {'input_path': self.input_path, 'paths_column': 'path', 'all_paths_columns': ['path'],
+                         'meta_columns': ['wav_converter_perf', 'vad_perf'], 'feature_columns': ['MFCC0', 'embedding_feature1'],
+                         'classification_columns': ['gender']}
+        self.df = pd.DataFrame({'path': ['a', 'b', 'c'], 'wav_converter_perf': [20, 30, 40], 'vad_perf': [1.2 , 5, 7],
+                               'MFCC0': [37.5, 36.6, 38.4], 'embedding_feature1': [80, 75, 72], 'gender': ['M', 'F', 'F']})
+        self.payload = ComponentPayload(self.input_path, self.metadata, self.df)
+
+    def test_init(self):
+        self.assertEqual(self.payload.metadata, self.metadata)
+        self.assertTrue(self.payload.df.equals(self.df))
+
+    def test_unpack(self):
+        self.assertEqual(self.payload.unpack(), (self.metadata, self.df))
+
+    def test_get_columns(self):
+        self.assertEqual(self.payload.get_columns(), ['path'])
+        self.assertEqual(self.payload.get_columns(all_paths_columns=True), ['path'])
+        self.assertEqual(self.payload.get_columns(meta_columns=True), ['path', 'wav_converter_perf', 'vad_perf'])
+        self.assertEqual(self.payload.get_columns(all_paths_columns=True, meta_columns=True), ['path', 'wav_converter_perf', 'vad_perf'])
+
+    def test_get_declared_columns(self):
+        self.assertTrue(self.payload.get_declared_columns(['feature_columns']).equals(self.df[['path', 'MFCC0', 'embedding_feature1']]))
+        self.assertTrue(self.payload.get_declared_columns(['classification_columns']).equals(self.df[['path', 'gender']]))
+        self.assertTrue(self.payload.get_declared_columns(['feature_columns', 'classification_columns']).equals(self.df[['path', 'MFCC0', 'embedding_feature1', 'gender']]))
diff --git a/tests/test_PipelineComponent.py b/tests/test_PipelineComponent.py
@@ -0,0 +1,34 @@
+import tempfile
+from unittest import TestCase
+import unittest
+import unittest.mock
+import io
+import pandas as pd
+from src.vanpy.core.PipelineComponent import PipelineComponent
+from src.vanpy.core.ComponentPayload import ComponentPayload
+
+class TestPipelineComponent(TestCase):
+    def setUp(self):
+        self.component_type = "example_component"
+        self.component_name = "example"
+        temp_dir = tempfile.mkdtemp()
+        self.config = {'example_component': {
+                           'example': {"log_each_x_records": 1, "items_in_paths_list": 10, "save_payload": True, "intermediate_payload_path": temp_dir}
+                        }
+                    }
+
+        class ImpPipelineComponent(PipelineComponent):
+            def process(self, input_payload: ComponentPayload) -> ComponentPayload:
+                pass
+        self.pipeline_component = ImpPipelineComponent(self.component_type, self.component_name, self.config)
+        self.input_payload = ComponentPayload(input_path=temp_dir, df=pd.DataFrame({"col1": [1, 2, 3], "col2": [4, 5, 6]}))
+
+    # @unittest.mock.patch(new_callable=io.StringIO)
+    # def test_latent_info_log(self, mock_stdout):
+    #     self.pipeline_component.latent_info_log("example message", 2)
+    #     self.assertEqual(mock_stdout.getvalue(), "example message")
+    #     self.assertEqual(self.pipeline_component.logger.info.call_count, 1)
+    #     self.pipeline_component.latent_info_log("example message", 3, last_item=True)
+    #     self.assertEqual(self.pipeline_component.logger.info.call_count, 2)
+    #     self.pipeline_component.latent_info_log("example message", 4)
+    #     self.assertEqual(self.pipeline_component.logger.info.call_count, 2)
diff --git a/tests/test_utils.py b/tests/test_utils.py

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+[build-system]`
	`2`	`+requires = ["setuptools>=42.0", "wheel"]`
	`3`	`+build-backend = "setuptools.build_meta"`
Original file line number	Diff line number	Diff line change
`@@ -1 +1 @@`
`1`		`-__version__ = '0.71.31'`
	`1`	`+__version__ = '0.71.33'`