Extract long running mujoco tests in separate job (#1436)

.codecov.yml

@@ -10,4 +10,4 @@ codecov:
     - "travis-ci.com"
   notify:
     wait_for_ci: yes
-    after_n_builds: 3
+    after_n_builds: 4

.travis.yml

@@ -41,6 +41,10 @@ jobs:
       name: "MuJoCo-based tests"
       env:
         - JOB_RUN_CMD="make ci-job-mujoco"
+    - if: type != pull_request OR head_repo = "rlworkgroup/garage"
+      name: "MuJoCo-based long running tests"
+      env:
+        - JOB_RUN_CMD="make ci-job-mujoco-long"
     - name: "Verify conda and pipenv installations"
       env: JOB_RUN_CMD="make ci-job-verify-envs"
     - if: type = cron

Makefile

@@ -28,18 +28,22 @@ ci-job-precommit: assert-docker docs
 ci-job-normal: assert-docker
 	[ ! -f $(MJKEY_PATH) ] || mv $(MJKEY_PATH) $(MJKEY_PATH).bak
 	pytest --cov=garage --cov-report=xml -m \
-	    'not nightly and not huge and not flaky and not large and not mujoco' --durations=20
+	    'not nightly and not huge and not flaky and not large and not mujoco and not mujoco_long' --durations=20
 	bash <(curl -s https://codecov.io/bash)
 
 ci-job-large: assert-docker
 	[ ! -f $(MJKEY_PATH) ] || mv $(MJKEY_PATH) $(MJKEY_PATH).bak
-	pytest --cov=garage --cov-report=xml -m 'large and not mujoco' --durations=20
+	pytest --cov=garage --cov-report=xml -m 'large and not flaky' --durations=20
 	bash <(curl -s https://codecov.io/bash)
 
 ci-job-mujoco: assert-docker
 	pytest --cov=garage --cov-report=xml -m 'mujoco and not flaky' --durations=20
 	bash <(curl -s https://codecov.io/bash)
 
+ci-job-mujoco-long: assert-docker
+	pytest --cov=garage --cov-report=xml -m 'mujoco_long and not flaky' --durations=20
+	bash <(curl -s https://codecov.io/bash)
+
 ci-job-nightly: assert-docker
 	pytest -m nightly
 

setup.cfg

@@ -32,6 +32,7 @@ markers =
     large
     serial
     mujoco
+    mujoco_long
 
 [yapf]
 based_on_style = pep8

tests/garage/tf/algos/test_ddpg.py

@@ -19,7 +19,7 @@
 class TestDDPG(TfGraphTestCase):
     """Tests for DDPG algorithm."""
 
-    @pytest.mark.mujoco
+    @pytest.mark.mujoco_long
     def test_ddpg_double_pendulum(self):
         """Test DDPG with Pendulum environment."""
         with LocalTFRunner(snapshot_config, sess=self.sess) as runner:
@@ -57,7 +57,7 @@ def test_ddpg_double_pendulum(self):
 
             env.close()
 
-    @pytest.mark.mujoco
+    @pytest.mark.mujoco_long
     def test_ddpg_pendulum(self):
         """Test DDPG with Pendulum environment.
 
@@ -98,7 +98,7 @@ def test_ddpg_pendulum(self):
 
             env.close()
 
-    @pytest.mark.mujoco
+    @pytest.mark.mujoco_long
     def test_ddpg_pendulum_with_decayed_weights(self):
         """Test DDPG with Pendulum environment and decayed weights.
 

tests/garage/tf/algos/test_ppo.py

@@ -212,7 +212,7 @@ def test_ppo_pendulum_continuous_baseline(self):
 
             env.close()
 
-    @pytest.mark.mujoco
+    @pytest.mark.mujoco_long
     def test_ppo_pendulum_recurrent_continuous_baseline(self):
         """Test PPO with Pendulum environment and recurrent policy."""
         with LocalTFRunner(snapshot_config) as runner:
@@ -248,7 +248,7 @@ def test_ppo_pendulum_recurrent_continuous_baseline(self):
 
 class TestPPOPendulumLSTM(TfGraphTestCase):
 
-    @pytest.mark.mujoco
+    @pytest.mark.mujoco_long
     def test_ppo_pendulum_lstm(self):
         """Test PPO with Pendulum environment and recurrent policy."""
         with LocalTFRunner(snapshot_config) as runner:
@@ -282,7 +282,7 @@ def test_ppo_pendulum_lstm(self):
 
 class TestPPOPendulumGRU(TfGraphTestCase):
 
-    @pytest.mark.mujoco
+    @pytest.mark.mujoco_long
     def test_ppo_pendulum_gru(self):
         """Test PPO with Pendulum environment and recurrent policy."""
         with LocalTFRunner(snapshot_config) as runner:

tests/garage/tf/algos/test_td3.py

@@ -17,7 +17,7 @@
 class TestTD3(TfGraphTestCase):
     """Tests for TD3 algo."""
 
-    @pytest.mark.mujoco
+    @pytest.mark.mujoco_long
     def test_td3_pendulum(self):
         """Test TD3 with Pendulum environment."""
         with LocalTFRunner(snapshot_config) as runner:

tests/garage/tf/algos/test_tnpg.py

@@ -12,7 +12,7 @@
 
 class TestTNPG(TfGraphTestCase):
 
-    @pytest.mark.mujoco
+    @pytest.mark.mujoco_long
     def test_tnpg_inverted_pendulum(self):
         """Test TNPG with InvertedPendulum-v2 environment."""
         with LocalTFRunner(snapshot_config, sess=self.sess) as runner:

tests/garage/tf/algos/test_trpo.py

@@ -39,7 +39,7 @@ def setup_method(self):
             regressor_args=dict(hidden_sizes=(32, 32)),
         )
 
-    @pytest.mark.mujoco
+    @pytest.mark.mujoco_long
     def test_trpo_pendulum(self):
         """Test TRPO with Pendulum environment."""
         with LocalTFRunner(snapshot_config, sess=self.sess) as runner:
@@ -69,7 +69,7 @@ def test_trpo_unknown_kl_constraint(self):
                 kl_constraint='random kl_constraint',
             )
 
-    @pytest.mark.mujoco
+    @pytest.mark.mujoco_long
     def test_trpo_soft_kl_constraint(self):
         """Test TRPO with unkown KL constraints."""
         with LocalTFRunner(snapshot_config, sess=self.sess) as runner:
@@ -85,7 +85,7 @@ def test_trpo_soft_kl_constraint(self):
             last_avg_ret = runner.train(n_epochs=10, batch_size=2048)
             assert last_avg_ret > 45
 
-    @pytest.mark.mujoco
+    @pytest.mark.mujoco_long
     def test_trpo_lstm_cartpole(self):
         with LocalTFRunner(snapshot_config, sess=self.sess) as runner:
             env = TfEnv(normalize(gym.make('CartPole-v1')))
@@ -110,7 +110,7 @@ def test_trpo_lstm_cartpole(self):
 
             env.close()
 
-    @pytest.mark.mujoco
+    @pytest.mark.mujoco_long
     def test_trpo_gru_cartpole(self):
         deterministic.set_seed(2)
         with LocalTFRunner(snapshot_config, sess=self.sess) as runner:

tests/garage/torch/algos/test_ddpg.py

@@ -18,7 +18,7 @@
 class TestDDPG:
     """Test class for DDPG."""
 
-    @pytest.mark.mujoco
+    @pytest.mark.mujoco_long
     def test_ddpg_double_pendulum(self):
         """Test DDPG with Pendulum environment."""
         deterministic.set_seed(0)
@@ -58,7 +58,7 @@ def test_ddpg_double_pendulum(self):
 
         env.close()
 
-    @pytest.mark.mujoco
+    @pytest.mark.mujoco_long
     def test_ddpg_pendulum(self):
         """Test DDPG with Pendulum environment.