Skip to content

Commit 0a551d7

Browse files
Mark2000Mark2000
committed
Issue #0: Release 1.0.1
1 parent cf5a415 commit 0a551d7

File tree

10 files changed

+5462
-764
lines changed

10 files changed

+5462
-764
lines changed

‎.gitignore

-1
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,6 @@ target/
7575
.ipynb_checkpoints
7676

7777
# PDFs and Images
78-
*.pdf
7978
*.jpg
8079

8180
# pyenv

‎CITATION.cff

+61-17
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,63 @@
11
cff-version: 1.2.0
2-
message: "If you use this software, please cite it as below."
2+
title: >-
3+
BSK-RL: Modular, High-Fidelity Reinforcement Learning
4+
Environments for Spacecraft Tasking
5+
message: >-
6+
If you use this software, please cite it using the
7+
metadata from this file.
8+
type: software
39
authors:
4-
- family-names: "Stephenson"
5-
given-names: "Mark"
6-
orcid: "https://orcid.org/0009-0004-3438-8127"
7-
- family-names: "Mantovani"
8-
given-names: "Lorenzzo"
9-
orcid: "https://orcid.org/0000-0001-7244-7491"
10-
- family-names: "Herrmann"
11-
given-names: "Adam"
12-
orcid: "https://orcid.org/0000-0001-6179-7728"
13-
- family-names: "Schaub"
14-
given-names: "Hanspeter"
15-
orcid: "https://orcid.org/0000-0003-0002-6035"
16-
title: "BSK-RL"
17-
version: 0.0.0
18-
date-released: 2023
19-
url: "https://github.com/AVSLab/bsk_rl/"
10+
- given-names: Mark
11+
family-names: Stephenson
12+
email: [email protected]
13+
affiliation: 'University of Colorado, Boulder'
14+
orcid: 'https://orcid.org/0009-0004-3438-8127'
15+
- given-names: Hanspeter
16+
family-names: Schaub
17+
orcid: 'https://orcid.org/0000-0003-0002-6035'
18+
affiliation: 'University of Colorado, Boulder'
19+
email: [email protected]
20+
identifiers:
21+
- type: url
22+
value: 'https://hanspeterschaub.info/Papers/Stephenson2024c.pdf'
23+
repository-code: 'https://github.com/AVSLab/bsk_rl/'
24+
url: 'https://avslab.github.io/bsk_rl/'
25+
abstract: >-
26+
Reinforcement learning (RL) is a highly adaptable
27+
framework for generating autonomous agents across a wide
28+
domain of problems. While RL has been successfully applied
29+
to highly complex, real-world systems, a significant
30+
amount of the literature studies abstractions and
31+
idealized versions of problems. This is especially the
32+
case for the field of spacecraft tasking, in which even
33+
traditional preplanning approaches tend to use highly
34+
simplified models of spacecraft dynamics and operations.
35+
When simplified methods are tested in a full-fidelity
36+
simulation, they often lead to conservative solutions that
37+
are suboptimal or aggressive solutions that are
38+
infeasible. As a result, there is a need for a
39+
high-fidelity spacecraft simulation environment to
40+
evaluate RL-based and other tasking algorithms. This paper
41+
introduces BSK-RL, an open-source Python package for
42+
creating and customizing reinforcement learning
43+
environments for spacecraft tasking problems. It combines
44+
Basilisk --- a high-speed and high-fidelity spacecraft
45+
simulation framework --- with abstractions of satellite
46+
tasks and operational objectives within the standard
47+
Gymnasium API wrapper for RL environments. The package is
48+
designed to meet the needs of RL and spacecraft operations
49+
researchers: Environment parameters are easily
50+
reproducible, customizable, and randomizable. Environments
51+
are highly modular: satellite state and action spaces can
52+
be specified, mission objectives and rewards can be
53+
defined, and the satellite dynamics and flight software
54+
can be configured, implicitly introducing operational
55+
limitations and safety constraints. Heterogeneous
56+
multi-agent environments can be created for more complex
57+
mission scenarios that consider communication and
58+
collaboration. Training and deployment using the package
59+
are demonstrated for an Earth-observing satellite with
60+
resource constraints.
61+
license: MIT
62+
version: 1.0.1
63+
date-released: '2024-08-27'

‎docs/build/doctrees/nbsphinx/examples/rllib_training.ipynb

+5,356-715
Large diffs are not rendered by default.

‎docs/source/_static/stephenson_bskrl_2024.pdf

568 KB
Binary file not shown.

‎docs/source/citation.rst

+11-8
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,24 @@
11
Citation
22
========
3-
If you use this code in your research, please cite the repository as follows:
3+
If you use this code in your research, please cite the :download:`IAC 2024 conference paper <_static/stephenson_bskrl_2024.pdf>`.
44

55
APA
66
---
77
.. code-block::
88
9-
Stephenson, M., Mantovani, L., Herrmann, A., & Schaub, H. BSK-RL (Version 0.0.0) [Computer software]. https://github.com/AVSLab/bsk_rl/
9+
M. Stephenson and H. Schaub, “BSK-RL: Modular, High-Fidelity Reinforcement Learning Environments for Spacecraft Tasking,” International Astronautical Congress, Milan, Italy, Oct. 14–18 2024.
1010
1111
BibTeX
1212
------
1313
.. code-block::
1414
15-
@software{
16-
Stephenson_BSK-RL,
17-
author = {Stephenson, Mark and Mantovani, Lorenzzo and Herrmann, Adam and Schaub, Hanspeter},
18-
title = {{BSK-RL}},
19-
url = {https://github.com/AVSLab/bsk_rl/},
20-
version = {0.0.0}
15+
@inproceedings{stephenson_bskrl_2024,
16+
title = {{{BSK-RL}}: {{Modular}}, {{High-Fidelity Reinforcement Learning Environments}} for {{Spacecraft Tasking}}},
17+
booktitle = {75th {{International Astronautical Congress}}},
18+
author = {Stephenson, Mark A and Schaub, Hanspeter},
19+
year = {2024},
20+
month = oct,
21+
publisher = {IAF},
22+
address = {Milan, Italy},
23+
langid = {english},
2124
}

‎docs/source/conf.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@
5353
}
5454
autodoc_typehints = "both"
5555
# nbsphinx_execute = "never"
56-
nbsphinx_allow_errors = True
56+
nbsphinx_allow_errors = False
5757

5858
# The language for content autogenerated by Sphinx. Refer to documentation
5959
# for a list of supported languages.

‎docs/source/index.rst

+3
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,9 @@ simulation framework, making the simulation environments high-fidelity and compu
2828
efficient. BSK-RL also includes a collection of utilities and examples
2929
for working with these environments.
3030

31+
A whitepaper on the design philosophy behind BSK-RL and an examble use case can be
32+
:download:`downloaded here <_static/stephenson_bskrl_2024.pdf>`.
33+
3134
Quickstart
3235
----------
3336
Installation

‎docs/source/release_notes.rst

+7-1
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,15 @@
11
Release Notes
22
=============
33

4+
Development Version
5+
-------------------
6+
*Release Date: MMM. DD, YYYY*
7+
8+
* None.
9+
410
Version 1.0.1
511
-------------
6-
*Release Date: MMM. DD, YYYY*
12+
*Release Date: Aug. 29, 2024*
713

814
* Change the :class:`~bsk_rl.ConstellationTasking` environment info dictionary to include
915
all non-agent information in ``info['__common__']``, which is expected by RLlib's

‎examples/rllib_training.ipynb

+22-20
Original file line numberDiff line numberDiff line change
@@ -13,8 +13,8 @@
1313
"\n",
1414
"<div class=\"alert alert-warning\">\n",
1515
"\n",
16-
"**Warning:** RLlib currently has a bug that results in an undesirable timeout which stops\n",
17-
"training. Check here to see if it has been resolved: https://github.com/ray-project/ray/pull/45147\n",
16+
"**Warning:** RLlib had a bug that results in an undesirable timeout which stops\n",
17+
"training. It has since been resolved: https://github.com/ray-project/ray/pull/45147\n",
1818
"\n",
1919
"</div>\n",
2020
"\n",
@@ -65,18 +65,18 @@
6565
" dict(prop=\"instrument_pointing_error\", norm=np.pi),\n",
6666
" dict(prop=\"solar_pointing_error\", norm=np.pi)\n",
6767
" ),\n",
68-
" obs.Eclipse(),\n",
6968
" obs.OpportunityProperties(\n",
7069
" dict(prop=\"opportunity_open\", norm=5700),\n",
7170
" dict(prop=\"opportunity_close\", norm=5700),\n",
7271
" type=\"ground_station\",\n",
7372
" n_ahead_observe=1,\n",
7473
" ),\n",
74+
" obs.Eclipse(norm=5700),\n",
7575
" obs.Time(),\n",
7676
" ]\n",
7777
" action_spec = [\n",
7878
" act.Scan(duration=180.0),\n",
79-
" act.Charge(duration=180.0),\n",
79+
" act.Charge(duration=120.0),\n",
8080
" act.Downlink(duration=60.0),\n",
8181
" act.Desat(duration=60.0),\n",
8282
" ]\n",
@@ -102,25 +102,25 @@
102102
" \"Scanner-1\",\n",
103103
" sat_args=dict(\n",
104104
" # Data\n",
105-
" dataStorageCapacity=5000 * 8e6, # MB to bits\n",
106-
" storageInit=lambda: np.random.uniform(0, 5000 * 8e6),\n",
107-
" instrumentBaudRate=0.5e6,\n",
108-
" transmitterBaudRate=-112e6,\n",
105+
" dataStorageCapacity=5000 * 8e6, # bits\n",
106+
" storageInit=lambda: np.random.uniform(0.0, 0.8) * 5000 * 8e6,\n",
107+
" instrumentBaudRate=0.5 * 8e6,\n",
108+
" transmitterBaudRate=-50 * 8e6,\n",
109109
" # Power\n",
110-
" batteryStorageCapacity=400 * 3600, # Wh to W*s\n",
111-
" storedCharge_Init=lambda: np.random.uniform(400 * 3600 * 0.2, 400 * 3600 * 0.8),\n",
112-
" basePowerDraw=-10.0,\n",
113-
" instrumentPowerDraw=-30.0,\n",
114-
" transmitterPowerDraw=-25.0,\n",
115-
" thrusterPowerDraw=-80.0,\n",
110+
" batteryStorageCapacity=200 * 3600, # W*s\n",
111+
" storedCharge_Init=lambda: np.random.uniform(0.3, 1.0) * 200 * 3600,\n",
112+
" basePowerDraw=-10.0, # W\n",
113+
" instrumentPowerDraw=-30.0, # W\n",
114+
" transmitterPowerDraw=-25.0, # W\n",
115+
" thrusterPowerDraw=-80.0, # W\n",
116+
" panelArea=0.25,\n",
116117
" # Attitude\n",
117118
" imageAttErrorRequirement=0.1,\n",
118119
" imageRateErrorRequirement=0.1,\n",
119-
" disturbance_vector=lambda: np.random.normal(scale=0.0001, size=3),\n",
120+
" disturbance_vector=lambda: np.random.normal(scale=0.0001, size=3), # N*m\n",
120121
" maxWheelSpeed=6000.0, # RPM\n",
121122
" wheelSpeeds=lambda: np.random.uniform(-3000, 3000, 3),\n",
122123
" desatAttitude=\"nadir\",\n",
123-
" nHat_B=np.array([0, 0, -1]), # Solar panel orientation\n",
124124
" )\n",
125125
")"
126126
]
@@ -139,7 +139,7 @@
139139
"metadata": {},
140140
"outputs": [],
141141
"source": [
142-
"duration = 2 * 5700.0 # About 2 orbits\n",
142+
"duration = 5 * 5700.0 # About 5 orbits\n",
143143
"env_args = dict(\n",
144144
" satellite=sat,\n",
145145
" scenario=scene.UniformNadirScanning(value_per_second=1/duration),\n",
@@ -210,10 +210,12 @@
210210
"from bsk_rl.utils.rllib import unpack_config\n",
211211
"from ray.rllib.algorithms.ppo import PPOConfig\n",
212212
"\n",
213+
"N_CPUS = 3\n",
214+
"\n",
213215
"training_args = dict(\n",
214216
" lr=0.00003,\n",
215217
" gamma=0.999,\n",
216-
" train_batch_size=250, # In practice, usually a bigger number\n",
218+
" train_batch_size=2500,\n",
217219
" num_sgd_iter=10,\n",
218220
" model=dict(fcnet_hiddens=[512, 512], vf_share_layers=False),\n",
219221
" lambda_=0.95,\n",
@@ -225,7 +227,7 @@
225227
"config = (\n",
226228
" PPOConfig()\n",
227229
" .training(**training_args)\n",
228-
" .env_runners(num_env_runners=2, sample_timeout_s=1000.0)\n",
230+
" .env_runners(num_env_runners=N_CPUS-1, sample_timeout_s=1000.0)\n",
229231
" .environment(\n",
230232
" env=unpack_config(SatelliteTasking),\n",
231233
" env_config=env_args,\n",
@@ -269,7 +271,7 @@
269271
"\n",
270272
"ray.init(\n",
271273
" ignore_reinit_error=True,\n",
272-
" num_cpus=3,\n",
274+
" num_cpus=N_CPUS,\n",
273275
" object_store_memory=2_000_000_000, # 2 GB\n",
274276
")\n",
275277
"\n",

‎pyproject.toml

+1-1
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
44

55
[project]
66
name = "bsk_rl"
7-
version = "1.0.0"
7+
version = "1.0.1"
88
authors = [
99
{ name = "Adam Herrmann", email = "[email protected]" },
1010
{ name = "Mark Stephenson", email = "[email protected]" },

0 commit comments

Comments
 (0)