Multiple fixes (#349)

CHANGES.rst

@@ -7,6 +7,12 @@ releases are available on `Anaconda.org
 <https://anaconda.org/OpenSourceEconomics/estimagic>`_.
 
 
+0.3.1
+-----
+
+- :gh:`349` fixes multiple small bugs and adds test cases for all of them
+  (:ghuser:`mpetrosian`, :ghuser:`janosg` and :ghuser:`timmens`)
+
 0.3.0
 -----
 

docs/source/index.rst

@@ -101,7 +101,7 @@ provides functionality to perform statistical inference on estimated parameters.
                             </p>
                         </div>
                     </div>
-                 </a>
+                </a>
             </div>
         </div>
     </div>

src/estimagic/estimation/estimate_ml.py

@@ -27,6 +27,7 @@
 from estimagic.parameters.conversion import get_converter
 from estimagic.shared.check_option_dicts import check_numdiff_options
 from estimagic.shared.check_option_dicts import check_optimization_options
+from estimagic.utilities import to_pickle
 
 
 def estimate_ml(
@@ -81,7 +82,7 @@ def estimate_ml(
         upper_bounds (pytree): As lower_bounds. Can be ``np.inf`` for parameters with
             no upper bound.
         constraints (list, dict): List with constraint dictionaries or single dict.
-            See .. _link: ../../docs/source/how_to_guides/how_to_use_constraints.ipynb
+            See :ref:`constraints`.
         logging (pathlib.Path, str or False): Path to sqlite3 file (which typically has
             the file extension ``.db``. If the file does not exist, it will be created.
             The dashboard can only be used when logging is used.
@@ -211,7 +212,7 @@ def estimate_ml(
     # Get the converter for params and function outputs
     # ==================================================================================
 
-    converter, flat_estimates = get_converter(
+    converter, internal_estimates = get_converter(
         func=loglike,
         params=estimates,
         constraints=constraints,
@@ -241,9 +242,9 @@ def func(x):
 
         jac_res = first_derivative(
             func=func,
-            params=flat_estimates.values,
-            lower_bounds=flat_estimates.lower_bounds,
-            upper_bounds=flat_estimates.upper_bounds,
+            params=internal_estimates.values,
+            lower_bounds=internal_estimates.lower_bounds,
+            upper_bounds=internal_estimates.upper_bounds,
             **numdiff_options,
         )
 
@@ -284,9 +285,9 @@ def func(x):
 
         hess_res = second_derivative(
             func=func,
-            params=flat_estimates.values,
-            lower_bounds=flat_estimates.lower_bounds,
-            upper_bounds=flat_estimates.upper_bounds,
+            params=internal_estimates.values,
+            lower_bounds=internal_estimates.lower_bounds,
+            upper_bounds=internal_estimates.upper_bounds,
             **numdiff_options,
         )
         int_hess = hess_res["derivative"]
@@ -335,7 +336,7 @@ def func(x):
         _internal_jacobian=int_jac,
         _internal_hessian=int_hess,
         _design_info=design_info,
-        _flat_params=flat_estimates,
+        _flat_params=internal_estimates,
         _has_constraints=constraints not in [None, []],
     )
 
@@ -615,7 +616,7 @@ def summary(
 
         summary = calculate_inference_quantities(
             estimates=self.params,
-            flat_estimates=self._flat_params,
+            internal_estimates=self._flat_params,
             free_cov=free_cov,
             ci_level=ci_level,
         )
@@ -737,3 +738,12 @@ def p_values(
         out = self._converter.params_from_internal(helper)
 
         return out
+
+    def to_pickle(self, path):
+        """Save the LikelihoodResult object to pickle.
+
+        Args:
+            path (str, pathlib.Path): A str or pathlib.path ending in .pkl or .pickle.
+
+        """
+        to_pickle(self, path=path)

src/estimagic/estimation/estimate_msm.py

@@ -39,6 +39,7 @@
 from estimagic.sensitivity.msm_sensitivity import calculate_sensitivity_to_weighting
 from estimagic.shared.check_option_dicts import check_numdiff_options
 from estimagic.shared.check_option_dicts import check_optimization_options
+from estimagic.utilities import to_pickle
 from pybaum import leaf_names
 from pybaum import tree_just_flatten
 
@@ -107,7 +108,7 @@ def estimate_msm(
             Note that "optimal" refers to the asymptotically optimal weighting matrix
             and is often not a good choice due to large finite sample bias.
         constraints (list, dict): List with constraint dictionaries or single dict.
-            See .. _link: ../../docs/source/how_to_guides/how_to_use_constraints.ipynb
+            See :ref:`constraints`.
         logging (pathlib.Path, str or False): Path to sqlite3 file (which typically has
             the file extension ``.db``. If the file does not exist, it will be created.
             The dashboard can only be used when logging is used.
@@ -259,7 +260,7 @@ def helper(params):
     else:
         func_eval = {"contributions": sim_mom_eval}
 
-    converter, flat_estimates = get_converter(
+    converter, internal_estimates = get_converter(
         func=helper,
         params=estimates,
         constraints=constraints,
@@ -289,9 +290,9 @@ def func(x):
 
         int_jac = first_derivative(
             func=func,
-            params=flat_estimates.values,
-            lower_bounds=flat_estimates.lower_bounds,
-            upper_bounds=flat_estimates.upper_bounds,
+            params=internal_estimates.values,
+            lower_bounds=internal_estimates.lower_bounds,
+            upper_bounds=internal_estimates.upper_bounds,
             **numdiff_options,
         )["derivative"]
 
@@ -320,7 +321,7 @@ def func(x):
     res = MomentsResult(
         params=estimates,
         weights=weights,
-        _flat_params=flat_estimates,
+        _flat_params=internal_estimates,
         _converter=converter,
         _internal_weights=internal_weights,
         _internal_moments_cov=internal_moments_cov,
@@ -659,7 +660,7 @@ def summary(
 
         summary = calculate_inference_quantities(
             estimates=self.params,
-            flat_estimates=self._flat_params,
+            internal_estimates=self._flat_params,
             free_cov=free_cov,
             ci_level=ci_level,
         )
@@ -930,3 +931,12 @@ def sensitivity(
             )
             raise ValueError(msg)
         return out
+
+    def to_pickle(self, path):
+        """Save the MomentsResult object to pickle.
+
+        Args:
+            path (str, pathlib.Path): A str or pathlib.path ending in .pkl or .pickle.
+
+        """
+        to_pickle(self, path=path)

src/estimagic/inference/shared.py

@@ -22,7 +22,7 @@ def transform_covariance(
             internal parameter vector. For background information about internal and
             external params see :ref:`implementation_of_constraints`.
         constraints (list): List with constraint dictionaries.
-            See .. _link: ../../docs/source/how_to_guides/how_to_use_constraints.ipynb
+            See :ref:`constraints`.
         n_samples (int): Number of samples used to transform the covariance matrix of
             the internal parameter vector into the covariance matrix of the external
             parameters.
@@ -78,13 +78,13 @@ def transform_covariance(
     return free_cov
 
 
-def calculate_inference_quantities(estimates, flat_estimates, free_cov, ci_level):
+def calculate_inference_quantities(estimates, internal_estimates, free_cov, ci_level):
     """Add standard errors, pvalues and confidence intervals to params.
 
     Args
         params (pytree): The input parameter pytree.
-        flat_estimates (FlatParams): NamedTuple with internal estimated parameter values
-            and names, lower_bounds and upper_bounds, and free_mask.
+        internal_estimates (FlatParams): NamedTuple with internal estimated parameter
+            values and names, lower_bounds and upper_bounds, and free_mask.
         free_cov (pd.DataFrame): Quadratic DataFrame containing the covariance matrix
             of the free parameters. If parameters were fixed (explicitly or by other
             constraints) the index is a subset of params.index. The columns are the same
@@ -101,14 +101,15 @@ def calculate_inference_quantities(estimates, flat_estimates, free_cov, ci_level
 
     """
     if not isinstance(free_cov, pd.DataFrame):
-        free_index = np.array(flat_estimates.names)[flat_estimates.free_mask]
+        free_index = np.array(internal_estimates.names)[internal_estimates.free_mask]
         free_cov = pd.DataFrame(data=free_cov, columns=free_index, index=free_index)
     ####################################################################################
     # Construct summary data frame for flat estimates
     ####################################################################################
+    registry = get_registry(extended=True)
 
-    df = pd.DataFrame(index=flat_estimates.names)
-    df["value"] = flat_estimates.values
+    df = pd.DataFrame(index=internal_estimates.names)
+    df["value"] = tree_just_flatten(estimates, registry=registry)
     df.loc[free_cov.index, "standard_error"] = np.sqrt(np.diag(free_cov))
 
     df["p_value"] = calculate_p_values(
@@ -134,13 +135,11 @@ def calculate_inference_quantities(estimates, flat_estimates, free_cov, ci_level
     # Map summary data into params tree structure
     ####################################################################################
 
-    registry = get_registry(extended=True)
-
     # create tree with values corresponding to indices of df
-    indices = tree_unflatten(estimates, flat_estimates.names, registry=registry)
+    indices = tree_unflatten(estimates, internal_estimates.names, registry=registry)
 
-    indices_flat = tree_just_flatten(indices)
     estimates_flat = tree_just_flatten(estimates)
+    indices_flat = tree_just_flatten(indices)
 
     # use index chunks in indices_flat to access the corresponding sub data frame of df,
     # and use the index information stored in estimates_flat to form the correct (multi)

src/estimagic/optimization/optimize.py

@@ -83,7 +83,7 @@ def maximize(
         soft_upper_bounds (pytree): As soft_lower_bounds.
         criterion_kwargs (dict): Additional keyword arguments for criterion
         constraints (list, dict): List with constraint dictionaries or single dict.
-            See .. _link: ../../docs/source/how_to_guides/how_to_use_constraints.ipynb
+            See :ref:`constraints`.
         algo_options (dict): Algorithm specific configuration of the optimization. See
             :ref:`list_of_algorithms` for supported options of each algorithm.
         derivative (callable): Function that calculates the first derivative
@@ -281,7 +281,7 @@ def minimize(
         soft_upper_bounds (pytree): As soft_lower_bounds.
         criterion_kwargs (dict): Additional keyword arguments for criterion
         constraints (list, dict): List with constraint dictionaries or single dict.
-            See .. _link: ../../docs/source/how_to_guides/how_to_use_constraints.ipynb
+            See :ref:`constraints`.
         algo_options (dict): Algorithm specific configuration of the optimization. See
             :ref:`list_of_algorithms` for supported options of each algorithm.
         derivative (callable): Function that calculates the first derivative

src/estimagic/optimization/optimize_result.py

@@ -6,6 +6,7 @@
 
 import numpy as np
 import pandas as pd
+from estimagic.utilities import to_pickle
 
 
 @dataclass
@@ -88,6 +89,15 @@ def __repr__(self):
 
         return msg
 
+    def to_pickle(self, path):
+        """Save the OptimizeResult object to pickle.
+
+        Args:
+            path (str, pathlib.Path): A str or pathlib.path ending in .pkl or .pickle.
+
+        """
+        to_pickle(self, path=path)
+
 
 def _format_convergence_report(report, algorithm):
     report = pd.DataFrame.from_dict(report)

src/estimagic/parameters/consolidate_constraints.py

@@ -208,7 +208,7 @@ def _consolidate_fixes_with_equality_constraints(
             assert (
                 len(valcounts) == 1
             ), "Equality constrained parameters cannot be fixed to different values."
-            fixed_value[eq["index"]] = valcounts.index[0]
+            fixed_value[eq["index"]] = valcounts[0]
 
     return fixed_value
 

src/estimagic/parameters/process_selectors.py

@@ -1,3 +1,4 @@
+import warnings
 from collections import Counter
 
 import numpy as np
@@ -53,7 +54,9 @@ def process_selectors(constraints, params, tree_converter, param_names):
             registry=registry,
         )
         try:
-            selected = evaluator(helper)
+            with warnings.catch_warnings():
+                warnings.simplefilter("ignore", category=pd.errors.PerformanceWarning)
+                selected = evaluator(helper)
         except (KeyboardInterrupt, SystemExit):
             raise
         except Exception as e:
@@ -94,11 +97,13 @@ def _get_selection_field(constraint, selector_case, params_case):
             "dataframe": {"locs", "queries", "selectors"},
             "numpy array": {"locs", "selectors"},
             "pytree": {"selectors"},
+            "series": {"locs", "selectors"},
         },
         "one selector": {
             "dataframe": {"loc", "query", "selector"},
             "numpy array": {"loc", "selector"},
             "pytree": {"selector"},
+            "series": {"loc", "selector"},
         },
     }
 
@@ -180,6 +185,8 @@ def evaluator(params):
 def _get_params_case(params):
     if isinstance(params, pd.DataFrame) and "value" in params:
         params_case = "dataframe"
+    elif isinstance(params, pd.Series):
+        params_case = "series"
     elif isinstance(params, np.ndarray):
         params_case = "numpy array"
     else:

src/estimagic/utilities.py

@@ -1,11 +1,16 @@
 import warnings
 from hashlib import sha1
 
+import cloudpickle
 import numpy as np
-from fuzzywuzzy import process as fw_process
+import pandas as pd
 from scipy.linalg import ldl
 from scipy.linalg import qr
 
+with warnings.catch_warnings():
+    warnings.simplefilter("ignore", category=UserWarning)
+    from fuzzywuzzy import process as fw_process
+
 
 def chol_params_to_lower_triangular_matrix(params):
     dim = number_of_triangular_elements_to_dimension(len(params))
@@ -125,7 +130,9 @@ def propose_alternatives(requested, possibilities, number=3):
 
     """
     number = min(number, len(possibilities))
-    proposals_w_probs = fw_process.extract(requested, possibilities, limit=number)
+    with warnings.catch_warnings():
+        warnings.simplefilter("ignore", category=UserWarning)
+        proposals_w_probs = fw_process.extract(requested, possibilities, limit=number)
     proposals = [proposal[0] for proposal in proposals_w_probs]
 
     return proposals
@@ -277,3 +284,12 @@ def calculate_trustregion_initial_radius(x):
     """
     x_norm = np.linalg.norm(x, ord=np.inf)
     return 0.1 * max(x_norm, 1)
+
+
+def to_pickle(obj, path):
+    with open(path, "wb") as buffer:
+        cloudpickle.dump(obj, buffer)
+
+
+def read_pickle(path):
+    return pd.read_pickle(path)

src/estimagic/visualization/estimation_table.py

@@ -285,7 +285,7 @@ def render_latex(
             r"""Proper LaTeX compilation requires the package siunitx and adding
                    \sisetup{
                        input-symbols            = (),
-                       table-align-text-post    = false
+                       table-align-text-post    = false,
                        group-digits             = false,
                     }
                     to your main tex file. To turn
@@ -322,6 +322,7 @@ def render_latex(
     default_options = {
         "multicol_align": "c",
         "hrules": True,
+        "siunitx": True,
         "column_format": "l" * n_levels + "S" * n_columns,
     }
     if render_options:

src/estimagic/visualization/slice_plot.py

@@ -155,7 +155,10 @@ def slice_plot(
     )
 
     # add NaNs where an evaluation failed
-    func_values = [val if not isinstance(val, str) else np.nan for val in func_values]
+    func_values = [
+        converter.func_to_internal(val) if not isinstance(val, str) else np.nan
+        for val in func_values
+    ]
 
     func_values += [converter.func_to_internal(func_eval)] * len(selected)
     for pos in selected: