From b2b1af62b96a4a8f1c1d9ffb2c36473aff603424 Mon Sep 17 00:00:00 2001 From: Lyndon Duong Date: Mon, 1 Feb 2021 16:45:49 -0500 Subject: [PATCH 01/15] init joss submission github action --- .github/workflows/draft-pdf.yml | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 .github/workflows/draft-pdf.yml diff --git a/.github/workflows/draft-pdf.yml b/.github/workflows/draft-pdf.yml new file mode 100644 index 00000000..229f8739 --- /dev/null +++ b/.github/workflows/draft-pdf.yml @@ -0,0 +1,29 @@ +on: + push: + branches: + - joss + pull_request: + branches: + - joss + +jobs: + paper: + runs-on: ubuntu-latest + name: Paper Draft + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Build draft PDF + uses: openjournals/openjournals-draft-action@master + with: + journal: joss + # This should be the path to the paper within your repo. + paper-path: /joss_paper/paper.md + - name: Upload + uses: actions/upload-artifact@v1 + with: + name: paper + # This is the output path where Pandoc will write the compiled + # PDF. Note, this should be the same directory as the input + # paper.md + path: /joss_paper/paper.pdf From 8012e9da95d8c277f161422deebd6bd9c6d804dd Mon Sep 17 00:00:00 2001 From: Lyndon D Date: Mon, 1 Feb 2021 16:47:24 -0500 Subject: [PATCH 02/15] joss paper wip --- joss_paper/paper.md | 76 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 joss_paper/paper.md diff --git a/joss_paper/paper.md b/joss_paper/paper.md new file mode 100644 index 00000000..bbab067f --- /dev/null +++ b/joss_paper/paper.md @@ -0,0 +1,76 @@ +--- +title: 'Plenoptic: synthesis methods for analyzing model representations' +tags: + - Python + - PyTorch + - neural networks + - computational neuroscience + - image synthesis +authors: + - name: Kathryn Bonnen + orcid: 0000-0002-9210-8275 + affiliation: 1, 2 + - name: William Broderick + orcid: 0000-0002-8999-9003 + affiliation: 1 + - name: Lyndon R. Duong + orcid: 0000-0003-0575-1033 + affiliation: 1 + - name: Pierre-Etienne Fiquet + orcid: 0000-0002-8301-2220 + affiliation: 1 + - name: Nikhil Parthasarathy + orcid: 0000-0003-2572-6492 + affiliation: 1 + - name: Eero P. Simoncelli + orcid: 000-0002-1206-527X + affiliation: 1, 2 +affiliations: + - name: Center for Neural Science, New York University, New York, NY, USA + index: 1 + - name: Center for Computational Neuroscience, Flatiron Institute, New York, NY, USA + index: 2 +date: April 2021 +bibliography: paper.bib +--- + +# Summary + + +twofold: 1) probe the internal representation of an arbitrary model, 2) provide a computational neuroscience. +generate new predictions about the human visual system. + +At its core, ``Plenoptic`` enables researchers to understand how a model views the world by means of image _synthesis_. +Take a deep convolutional neural network, for example; an input image ellicits a pattern of activations within each layer of the network. +Given an image and its associated pattern of model activations, a researcher might ask the following questions: 1) are there different images that can evoke an identical response, and thus be perceived as identical to the model [@metamers]; 2)[@] +Using modern operimization tools (e.g. automatic differentiation), ``Plenoptic`` enables researchers to answer each of these questions. + +``Plenoptic`` builds primarily off of ``PyTorch`` [@pytorch], a Python machine learning library popular in the research community due to its rapid prototyping capability. With ``Plenoptic``, users can build and train models in ``PyTorch``, then use ``Plenoptic`` synthesis methods to assess their internal representations. +Our library is easily extensible, and allows for great flexibility to those who wish to develop or test their own synthesis methods. +Within the library, we also provide an extensive suite of ``PyTorch``-implemented models and activation functions canonical to computational neuroscience. + +Many of the methods in ``Plenoptic`` have been developed and used across several studies [@]; however, analyses in these studies used disparate languages and frameworks, and some have yet to be made publicly available. +Here, we have reimplemented the methods central to each of these studies, and unified them under a single, fully-documented API. +Our library includes several Jupyter notebook tutorials designed to be accessible to researchers in the fields of machine learning, and computational neuroscience, and perceptual science. +``Plenoptic`` provides an exciting avenue for researchers to probe their models to gain a deeper understanding of their internal representations. + +# Acknowledgements + +KB, WB, LRD, PEF, and NP each contributed equally to this work. +EPS was funded by the Howard Hughes Medical Institute. EPS and KB were funded by Simons Institute. + +For a quick reference, the following citation commands can be used: +- `@author:2001` -> "Author et al. (2001)" +- `[@author:2001]` -> "(Author et al., 2001)" +- `[@author1:2001; @author2:2001]` -> "(Author1 et al., 2001; Author2 et al., 2002)" + +# References + +@eigendistortions +@geodesics +@pyramid +@metamers +@madcompetition +@pytorch +@portillasimoncelli + From 62e82f03364fa6096cbb3c09752e3b1527844446 Mon Sep 17 00:00:00 2001 From: Lyndon D Date: Mon, 1 Feb 2021 16:51:45 -0500 Subject: [PATCH 03/15] github action name change --- .github/workflows/draft-pdf.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/draft-pdf.yml b/.github/workflows/draft-pdf.yml index 229f8739..c7d627b4 100644 --- a/.github/workflows/draft-pdf.yml +++ b/.github/workflows/draft-pdf.yml @@ -1,3 +1,4 @@ +name: joss on: push: branches: From 7d607c283c9ba58b21e84288abc751b75d205062 Mon Sep 17 00:00:00 2001 From: Lyndon D Date: Mon, 1 Feb 2021 16:57:15 -0500 Subject: [PATCH 04/15] minor change to directory --- .github/workflows/draft-pdf.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/draft-pdf.yml b/.github/workflows/draft-pdf.yml index c7d627b4..7c088ba7 100644 --- a/.github/workflows/draft-pdf.yml +++ b/.github/workflows/draft-pdf.yml @@ -19,7 +19,7 @@ jobs: with: journal: joss # This should be the path to the paper within your repo. - paper-path: /joss_paper/paper.md + paper-path: joss_paper/paper.md - name: Upload uses: actions/upload-artifact@v1 with: @@ -27,4 +27,4 @@ jobs: # This is the output path where Pandoc will write the compiled # PDF. Note, this should be the same directory as the input # paper.md - path: /joss_paper/paper.pdf + path: joss_paper/paper.pdf From 9387a764841b43cbb72307f9017d403aa62347f5 Mon Sep 17 00:00:00 2001 From: Lyndon D Date: Mon, 1 Feb 2021 17:12:49 -0500 Subject: [PATCH 05/15] prelim bibliography --- joss_paper/paper.bib | 76 ++++++++++++++++++++++++++++++++++++++++++++ joss_paper/paper.md | 26 ++++++--------- 2 files changed, 85 insertions(+), 17 deletions(-) create mode 100644 joss_paper/paper.bib diff --git a/joss_paper/paper.bib b/joss_paper/paper.bib new file mode 100644 index 00000000..27d0bd4c --- /dev/null +++ b/joss_paper/paper.bib @@ -0,0 +1,76 @@ + +@inproceedings{simoncelli_steerable_1995, + title = {The steerable pyramid: a flexible architecture for multi-scale derivative computation}, + volume = {3}, + isbn = {0-7803-3122-2}, + doi = {10.1109/ICIP.1995.537667}, + booktitle = {Proceedings., {International} {Conference} on {Image} {Processing}}, + publisher = {IEEE Comput. Soc. Press}, + author = {Simoncelli, E.P. and Freeman, W.T.}, + year = {1995}, + pages = {444--447}, +} + +@article{henaff_geodesics_2015, + title = {Geodesics of learned representations}, + author = {Hénaff, Olivier J. and Simoncelli, Eero P.}, + month = nov, + year = {2015}, + pages = {1--10}, +} + +@article{berardino_eigen-distortions_2017, + title = {Eigen-{Distortions} of {Hierarchical} {Representations}}, + journal = {arXiv:1710.02266 [cs]}, + author = {Berardino, Alexander and Ballé, Johannes and Laparra, Valero and Simoncelli, Eero P.}, + month = oct, + year = {2017}, + note = {arXiv: 1710.02266}, + keywords = {Computer Science - Computer Vision and Pattern Recognition}, +} + +@article{freeman_metamers_2011, + title = {Metamers of the ventral stream}, + volume = {14}, + issn = {1097-6256, 1546-1726}, + doi = {10.1038/nn.2889}, + language = {en}, + number = {9}, + urldate = {2019-09-25}, + journal = {Nature Neuroscience}, + author = {Freeman, Jeremy and Simoncelli, Eero P}, + month = sep, + year = {2011}, + pages = {1195--1201}, +} + +@article{paszke_pytorch_2019, + title = {{PyTorch}: {An} {Imperative} {Style}, {High}-{Performance} {Deep} {Learning} {Library}}, + language = {en}, + author = {Paszke, Adam and Gross, Sam and Massa, Francisco and Lerer, Adam and Bradbury, James and Chanan, Gregory and Killeen, Trevor and Lin, Zeming and Gimelshein, Natalia}, + year = {2019} + pages = {12}, +} + +@article{wang_maximum_2008, + title = {Maximum differentiation ({MAD}) competition: {A} methodology for comparing computational models of perceptual quantities}, + volume = {8}, + issn = {1534-7362}, + shorttitle = {Maximum differentiation ({MAD}) competition}, + doi = {10.1167/8.12.8}, + language = {en}, + number = {12}, + urldate = {2019-12-28}, + journal = {Journal of Vision}, + author = {Wang, Z. and Simoncelli, E. P.}, + month = sep, + year = {2008}, + pages = {8--8}, +} + +@article{portilla_parametric_2000, + title = {A {Parametric} {Texture} {Model} {Based} on {Joint} {Statistics} of {Complex} {Wavelet} {Coefficients}}, + language = {en}, + author = {Portilla, Javier and Simoncelli, Eero P}, + pages = {23}, +} diff --git a/joss_paper/paper.md b/joss_paper/paper.md index bbab067f..8f009896 100644 --- a/joss_paper/paper.md +++ b/joss_paper/paper.md @@ -37,19 +37,11 @@ bibliography: paper.bib # Summary -twofold: 1) probe the internal representation of an arbitrary model, 2) provide a computational neuroscience. -generate new predictions about the human visual system. - -At its core, ``Plenoptic`` enables researchers to understand how a model views the world by means of image _synthesis_. -Take a deep convolutional neural network, for example; an input image ellicits a pattern of activations within each layer of the network. -Given an image and its associated pattern of model activations, a researcher might ask the following questions: 1) are there different images that can evoke an identical response, and thus be perceived as identical to the model [@metamers]; 2)[@] -Using modern operimization tools (e.g. automatic differentiation), ``Plenoptic`` enables researchers to answer each of these questions. - -``Plenoptic`` builds primarily off of ``PyTorch`` [@pytorch], a Python machine learning library popular in the research community due to its rapid prototyping capability. With ``Plenoptic``, users can build and train models in ``PyTorch``, then use ``Plenoptic`` synthesis methods to assess their internal representations. +``Plenoptic`` builds primarily off of ``PyTorch`` [@paszke_pytorch_2019], a Python machine learning library popular in the research community due to its rapid prototyping capability. With ``Plenoptic``, users can build and train models in ``PyTorch``, then use ``Plenoptic`` synthesis methods to assess their internal representations. Our library is easily extensible, and allows for great flexibility to those who wish to develop or test their own synthesis methods. Within the library, we also provide an extensive suite of ``PyTorch``-implemented models and activation functions canonical to computational neuroscience. -Many of the methods in ``Plenoptic`` have been developed and used across several studies [@]; however, analyses in these studies used disparate languages and frameworks, and some have yet to be made publicly available. +Many of the methods in ``Plenoptic`` have been developed and used across several studies; however, analyses in these studies used disparate languages and frameworks, and some have yet to be made publicly available. Here, we have reimplemented the methods central to each of these studies, and unified them under a single, fully-documented API. Our library includes several Jupyter notebook tutorials designed to be accessible to researchers in the fields of machine learning, and computational neuroscience, and perceptual science. ``Plenoptic`` provides an exciting avenue for researchers to probe their models to gain a deeper understanding of their internal representations. @@ -66,11 +58,11 @@ For a quick reference, the following citation commands can be used: # References -@eigendistortions -@geodesics -@pyramid -@metamers -@madcompetition -@pytorch -@portillasimoncelli +@berardino_eigen-distortions_2017 +@henaff_geodesics_2015 +@simoncelli_steerable_1995 +@freeman_metamers_2011 +@wang_maximum_2008 +@paszke_pytorch_2019 +@portilla_parametric_2000 From b6bc3f4afdcb4f2fa510e65e2e90f2d088b1209b Mon Sep 17 00:00:00 2001 From: Lyndon D Date: Mon, 1 Feb 2021 17:16:31 -0500 Subject: [PATCH 06/15] .bib typo --- joss_paper/paper.bib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/joss_paper/paper.bib b/joss_paper/paper.bib index 27d0bd4c..94e582e1 100644 --- a/joss_paper/paper.bib +++ b/joss_paper/paper.bib @@ -48,7 +48,7 @@ @article{paszke_pytorch_2019 title = {{PyTorch}: {An} {Imperative} {Style}, {High}-{Performance} {Deep} {Learning} {Library}}, language = {en}, author = {Paszke, Adam and Gross, Sam and Massa, Francisco and Lerer, Adam and Bradbury, James and Chanan, Gregory and Killeen, Trevor and Lin, Zeming and Gimelshein, Natalia}, - year = {2019} + year = {2019}, pages = {12}, } From 2b57326f98a16cda122768a05705ca72d64f8939 Mon Sep 17 00:00:00 2001 From: Lyndon D Date: Mon, 1 Feb 2021 20:28:27 -0500 Subject: [PATCH 07/15] fix references --- joss_paper/paper.md | 8 +++-- joss_paper/{paper.bib => references.bib} | 44 +++++++++++++----------- 2 files changed, 29 insertions(+), 23 deletions(-) rename joss_paper/{paper.bib => references.bib} (52%) diff --git a/joss_paper/paper.md b/joss_paper/paper.md index 8f009896..c2d5c72d 100644 --- a/joss_paper/paper.md +++ b/joss_paper/paper.md @@ -31,7 +31,7 @@ affiliations: - name: Center for Computational Neuroscience, Flatiron Institute, New York, NY, USA index: 2 date: April 2021 -bibliography: paper.bib +bibliography: references.bib --- # Summary @@ -46,9 +46,13 @@ Here, we have reimplemented the methods central to each of these studies, and un Our library includes several Jupyter notebook tutorials designed to be accessible to researchers in the fields of machine learning, and computational neuroscience, and perceptual science. ``Plenoptic`` provides an exciting avenue for researchers to probe their models to gain a deeper understanding of their internal representations. +# Statement of Need + +# Overview + # Acknowledgements -KB, WB, LRD, PEF, and NP each contributed equally to this work. +KB, WB, LRD, PEF, and NP each contributed equally to this work; and names are listed alphabetically. EPS was funded by the Howard Hughes Medical Institute. EPS and KB were funded by Simons Institute. For a quick reference, the following citation commands can be used: diff --git a/joss_paper/paper.bib b/joss_paper/references.bib similarity index 52% rename from joss_paper/paper.bib rename to joss_paper/references.bib index 94e582e1..8925b091 100644 --- a/joss_paper/paper.bib +++ b/joss_paper/references.bib @@ -4,28 +4,29 @@ @inproceedings{simoncelli_steerable_1995 volume = {3}, isbn = {0-7803-3122-2}, doi = {10.1109/ICIP.1995.537667}, - booktitle = {Proceedings., {International} {Conference} on {Image} {Processing}}, + booktitle = {Proceedings., International Conference on Image Processing}, publisher = {IEEE Comput. Soc. Press}, - author = {Simoncelli, E.P. and Freeman, W.T.}, + author = {Simoncelli, Eero P. and Freeman, William T.}, year = {1995}, pages = {444--447}, } -@article{henaff_geodesics_2015, - title = {Geodesics of learned representations}, - author = {Hénaff, Olivier J. and Simoncelli, Eero P.}, - month = nov, - year = {2015}, - pages = {1--10}, +@article{henaff_geodesics_2016, + author = {{H{\'e}naff}, Olivier J. and {Simoncelli}, Eero P.}, + title = "{Geodesics of learned representations}", + journal = {International Conference on Learning Representations}, + year = 2016, + eid = {arXiv:1511.06394}, + eprint = {1511.06394}, } @article{berardino_eigen-distortions_2017, - title = {Eigen-{Distortions} of {Hierarchical} {Representations}}, - journal = {arXiv:1710.02266 [cs]}, + title = {Eigen-Distortions of Hierarchical Representations}, + journal = {Adv. Neural Information Processing Systems}, author = {Berardino, Alexander and Ballé, Johannes and Laparra, Valero and Simoncelli, Eero P.}, - month = oct, + month = dec, year = {2017}, - note = {arXiv: 1710.02266}, + pages = {3530-3539} keywords = {Computer Science - Computer Vision and Pattern Recognition}, } @@ -38,14 +39,14 @@ @article{freeman_metamers_2011 number = {9}, urldate = {2019-09-25}, journal = {Nature Neuroscience}, - author = {Freeman, Jeremy and Simoncelli, Eero P}, + author = {Freeman, Jeremy and Simoncelli, Eero P.}, month = sep, year = {2011}, pages = {1195--1201}, } @article{paszke_pytorch_2019, - title = {{PyTorch}: {An} {Imperative} {Style}, {High}-{Performance} {Deep} {Learning} {Library}}, + title = {PyTorch: An Imperative Style, High-Performance Deep Learning Library}, language = {en}, author = {Paszke, Adam and Gross, Sam and Massa, Francisco and Lerer, Adam and Bradbury, James and Chanan, Gregory and Killeen, Trevor and Lin, Zeming and Gimelshein, Natalia}, year = {2019}, @@ -53,24 +54,25 @@ @article{paszke_pytorch_2019 } @article{wang_maximum_2008, - title = {Maximum differentiation ({MAD}) competition: {A} methodology for comparing computational models of perceptual quantities}, + title = {Maximum differentiation (MAD) competition: A methodology for comparing computational models of perceptual quantities}, volume = {8}, issn = {1534-7362}, - shorttitle = {Maximum differentiation ({MAD}) competition}, + shorttitle = {Maximum differentiation (MAD) competition}, doi = {10.1167/8.12.8}, language = {en}, number = {12}, - urldate = {2019-12-28}, journal = {Journal of Vision}, - author = {Wang, Z. and Simoncelli, E. P.}, + author = {Wang, Zhou and Simoncelli, Eero P.}, month = sep, year = {2008}, pages = {8--8}, } @article{portilla_parametric_2000, - title = {A {Parametric} {Texture} {Model} {Based} on {Joint} {Statistics} of {Complex} {Wavelet} {Coefficients}}, + title = {A Parametric Texture Model Based on Joint Statistics of Complex Wavelet Coefficients}, language = {en}, - author = {Portilla, Javier and Simoncelli, Eero P}, - pages = {23}, + journal = {International Journal of Computer Vision}, + year = {2000}, + author = {Portilla, Javier and Simoncelli, Eero P.}, + pages = {49-71}, } From 999216046b2ef8fd9abd8b622ce63e4a41cf0d37 Mon Sep 17 00:00:00 2001 From: Lyndon D Date: Mon, 1 Feb 2021 20:47:53 -0500 Subject: [PATCH 08/15] fix refs --- joss_paper/references.bib | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/joss_paper/references.bib b/joss_paper/references.bib index 8925b091..1f3d863f 100644 --- a/joss_paper/references.bib +++ b/joss_paper/references.bib @@ -17,7 +17,6 @@ @article{henaff_geodesics_2016 journal = {International Conference on Learning Representations}, year = 2016, eid = {arXiv:1511.06394}, - eprint = {1511.06394}, } @article{berardino_eigen-distortions_2017, @@ -65,7 +64,7 @@ @article{wang_maximum_2008 author = {Wang, Zhou and Simoncelli, Eero P.}, month = sep, year = {2008}, - pages = {8--8}, + pages = {1-13}, } @article{portilla_parametric_2000, From 56074c1bb7694c38554cef738e64d53b341840a8 Mon Sep 17 00:00:00 2001 From: Lyndon D Date: Mon, 1 Feb 2021 20:49:40 -0500 Subject: [PATCH 09/15] fix refs --- joss_paper/references.bib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/joss_paper/references.bib b/joss_paper/references.bib index 1f3d863f..4ac595ed 100644 --- a/joss_paper/references.bib +++ b/joss_paper/references.bib @@ -25,7 +25,7 @@ @article{berardino_eigen-distortions_2017 author = {Berardino, Alexander and Ballé, Johannes and Laparra, Valero and Simoncelli, Eero P.}, month = dec, year = {2017}, - pages = {3530-3539} + pages = {3530-3539}, keywords = {Computer Science - Computer Vision and Pattern Recognition}, } From 253d39f29bdd46fe38596fea10ce0d2b66a78f51 Mon Sep 17 00:00:00 2001 From: Lyndon Duong Date: Tue, 10 Jan 2023 10:16:55 -0500 Subject: [PATCH 10/15] edit authors and text --- joss_paper/paper.md | 41 ++++++++++++++++++++++------------------- 1 file changed, 22 insertions(+), 19 deletions(-) diff --git a/joss_paper/paper.md b/joss_paper/paper.md index c2d5c72d..9e03d0a3 100644 --- a/joss_paper/paper.md +++ b/joss_paper/paper.md @@ -1,5 +1,5 @@ --- -title: 'Plenoptic: synthesis methods for analyzing model representations' +title: 'Plenoptic.py: Synthesizing model-optimized visual stimuli' tags: - Python - PyTorch @@ -10,7 +10,7 @@ authors: - name: Kathryn Bonnen orcid: 0000-0002-9210-8275 affiliation: 1, 2 - - name: William Broderick + - name: William F. Broderick orcid: 0000-0002-8999-9003 affiliation: 1 - name: Lyndon R. Duong @@ -22,6 +22,12 @@ authors: - name: Nikhil Parthasarathy orcid: 0000-0003-2572-6492 affiliation: 1 + - name: Xinyuan Zhao + orcid: 0000-0003-2572-6492 + affiliation: 1 + - name: Thomas E. Yerxa + orcid: 0000-0003-2572-6492 + affiliation: 1 - name: Eero P. Simoncelli orcid: 000-0002-1206-527X affiliation: 1, 2 @@ -30,32 +36,30 @@ affiliations: index: 1 - name: Center for Computational Neuroscience, Flatiron Institute, New York, NY, USA index: 2 -date: April 2021 +date: January 2023 bibliography: references.bib --- # Summary - -``Plenoptic`` builds primarily off of ``PyTorch`` [@paszke_pytorch_2019], a Python machine learning library popular in the research community due to its rapid prototyping capability. With ``Plenoptic``, users can build and train models in ``PyTorch``, then use ``Plenoptic`` synthesis methods to assess their internal representations. -Our library is easily extensible, and allows for great flexibility to those who wish to develop or test their own synthesis methods. -Within the library, we also provide an extensive suite of ``PyTorch``-implemented models and activation functions canonical to computational neuroscience. - -Many of the methods in ``Plenoptic`` have been developed and used across several studies; however, analyses in these studies used disparate languages and frameworks, and some have yet to be made publicly available. -Here, we have reimplemented the methods central to each of these studies, and unified them under a single, fully-documented API. -Our library includes several Jupyter notebook tutorials designed to be accessible to researchers in the fields of machine learning, and computational neuroscience, and perceptual science. -``Plenoptic`` provides an exciting avenue for researchers to probe their models to gain a deeper understanding of their internal representations. - -# Statement of Need - -# Overview +In sensory perception and neuroscience, new computational models are most often tested and compared in terms of their ability to fit existing data sets. +However, experimental data are inherently limited in size, quality, and type, and complex models often saturate their explainable variance. +Moreover, it is often difficult to use models to guide the development of future experiments. +Here, building on ideas for optimal experimental stimulus selection (e.g., QUEST, Watson and Pelli, 1983), we present "Plenoptic", a python software library for generating visual stimuli optimized for testing or comparing models. +Plenoptic provides a unified framework containing four previously-published synthesis methods -- model metamers (Freeman and Simoncelli, 2011), Maximum Differentiation (MAD) competition (Wang and Simoncelli, 2008), eigen-distortions (Berardino et al. 2017), and representational geodesics (Hénaff and Simoncelli, 2015) -- each of which offers visualization of model representations, and generation of images that can be used to experimentally test alignment with the human visual system. +Plenoptic leverages modern machine-learning methods to enable application of these synthesis methods to any computational model that satisfies a small set of common requirements. +The most important of these is that the model must be image-computable, implemented in PyTorch, and end-to-end differentiable. +The package includes examples of several low- and mid-level visual models, as well as a set of perceptual quality metrics. +Plenoptic is open source, tested, documented, and extensible, allowing the broader research community to contribute new examples and methods. +In summary, Plenoptic leverages machine learning tools to tighten the scientific hypothesis-testing loop, facilitating investigation of human visual representations. # Acknowledgements -KB, WB, LRD, PEF, and NP each contributed equally to this work; and names are listed alphabetically. -EPS was funded by the Howard Hughes Medical Institute. EPS and KB were funded by Simons Institute. +All authors contributed equally to this work; and names are listed alphabetically. +EPS and KB were funded by Simons Institute. For a quick reference, the following citation commands can be used: + - `@author:2001` -> "Author et al. (2001)" - `[@author:2001]` -> "(Author et al., 2001)" - `[@author1:2001; @author2:2001]` -> "(Author1 et al., 2001; Author2 et al., 2002)" @@ -69,4 +73,3 @@ For a quick reference, the following citation commands can be used: @wang_maximum_2008 @paszke_pytorch_2019 @portilla_parametric_2000 - From 2990208b300f5398a8cd0b2ea34687ff4adc7847 Mon Sep 17 00:00:00 2001 From: Lyndon Duong Date: Tue, 10 Jan 2023 10:25:31 -0500 Subject: [PATCH 11/15] edit authors --- joss_paper/paper.md | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/joss_paper/paper.md b/joss_paper/paper.md index 9e03d0a3..09ecf07f 100644 --- a/joss_paper/paper.md +++ b/joss_paper/paper.md @@ -8,25 +8,31 @@ tags: - image synthesis authors: - name: Kathryn Bonnen + equal-contrib: true orcid: 0000-0002-9210-8275 - affiliation: 1, 2 + affiliation: 3 - name: William F. Broderick + equal-contrib: true orcid: 0000-0002-8999-9003 affiliation: 1 - name: Lyndon R. Duong + equal-contrib: true orcid: 0000-0003-0575-1033 affiliation: 1 - name: Pierre-Etienne Fiquet + equal-contrib: true orcid: 0000-0002-8301-2220 affiliation: 1 - name: Nikhil Parthasarathy - orcid: 0000-0003-2572-6492 - affiliation: 1 - - name: Xinyuan Zhao + equal-contrib: true orcid: 0000-0003-2572-6492 affiliation: 1 - name: Thomas E. Yerxa - orcid: 0000-0003-2572-6492 + equal-contrib: true + orcid: 0000-0003-2687-0816 + affiliation: 1 + - name: Xinyuan Zhao + equal-contrib: true affiliation: 1 - name: Eero P. Simoncelli orcid: 000-0002-1206-527X @@ -36,6 +42,8 @@ affiliations: index: 1 - name: Center for Computational Neuroscience, Flatiron Institute, New York, NY, USA index: 2 + - name: School of Optometry, Indiana University, Bloomington, IN, USA + index: 3 date: January 2023 bibliography: references.bib --- @@ -53,9 +61,10 @@ The package includes examples of several low- and mid-level visual models, as we Plenoptic is open source, tested, documented, and extensible, allowing the broader research community to contribute new examples and methods. In summary, Plenoptic leverages machine learning tools to tighten the scientific hypothesis-testing loop, facilitating investigation of human visual representations. +# Statement of need + # Acknowledgements -All authors contributed equally to this work; and names are listed alphabetically. EPS and KB were funded by Simons Institute. For a quick reference, the following citation commands can be used: From 543c2c3a2fa3f745d8ee1d682fa87ac1643c1e70 Mon Sep 17 00:00:00 2001 From: Lyndon Duong Date: Tue, 10 Jan 2023 10:56:14 -0500 Subject: [PATCH 12/15] edit affiliations --- joss_paper/paper.md | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/joss_paper/paper.md b/joss_paper/paper.md index 09ecf07f..f0feee0e 100644 --- a/joss_paper/paper.md +++ b/joss_paper/paper.md @@ -10,39 +10,39 @@ authors: - name: Kathryn Bonnen equal-contrib: true orcid: 0000-0002-9210-8275 - affiliation: 3 + affiliation: 1 - name: William F. Broderick equal-contrib: true orcid: 0000-0002-8999-9003 - affiliation: 1 + affiliation: 2 - name: Lyndon R. Duong equal-contrib: true orcid: 0000-0003-0575-1033 - affiliation: 1 + affiliation: 2 - name: Pierre-Etienne Fiquet equal-contrib: true orcid: 0000-0002-8301-2220 - affiliation: 1 + affiliation: 2 - name: Nikhil Parthasarathy equal-contrib: true orcid: 0000-0003-2572-6492 - affiliation: 1 + affiliation: 2 - name: Thomas E. Yerxa equal-contrib: true orcid: 0000-0003-2687-0816 - affiliation: 1 + affiliation: 2 - name: Xinyuan Zhao equal-contrib: true - affiliation: 1 + affiliation: 2 - name: Eero P. Simoncelli orcid: 000-0002-1206-527X - affiliation: 1, 2 + affiliation: "2, 3" affiliations: - - name: Center for Neural Science, New York University, New York, NY, USA + - name: School of Optometry, Indiana University, Bloomington, IN, USA index: 1 - - name: Center for Computational Neuroscience, Flatiron Institute, New York, NY, USA + - name: Center for Neural Science, New York University, New York, NY, USA index: 2 - - name: School of Optometry, Indiana University, Bloomington, IN, USA + - name: Center for Computational Neuroscience, Flatiron Institute, New York, NY, USA index: 3 date: January 2023 bibliography: references.bib From 8bacd5aca41495a656a50982d24a852d99f294b8 Mon Sep 17 00:00:00 2001 From: Lyndon Duong Date: Tue, 10 Jan 2023 11:00:11 -0500 Subject: [PATCH 13/15] references --- joss_paper/paper.md | 2 +- joss_paper/references.bib | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/joss_paper/paper.md b/joss_paper/paper.md index f0feee0e..e31cc55b 100644 --- a/joss_paper/paper.md +++ b/joss_paper/paper.md @@ -56,7 +56,7 @@ Moreover, it is often difficult to use models to guide the development of future Here, building on ideas for optimal experimental stimulus selection (e.g., QUEST, Watson and Pelli, 1983), we present "Plenoptic", a python software library for generating visual stimuli optimized for testing or comparing models. Plenoptic provides a unified framework containing four previously-published synthesis methods -- model metamers (Freeman and Simoncelli, 2011), Maximum Differentiation (MAD) competition (Wang and Simoncelli, 2008), eigen-distortions (Berardino et al. 2017), and representational geodesics (Hénaff and Simoncelli, 2015) -- each of which offers visualization of model representations, and generation of images that can be used to experimentally test alignment with the human visual system. Plenoptic leverages modern machine-learning methods to enable application of these synthesis methods to any computational model that satisfies a small set of common requirements. -The most important of these is that the model must be image-computable, implemented in PyTorch, and end-to-end differentiable. +The most important of these is that the model must be image-computable, implemented in PyTorch (Paszke et al. 2019), and end-to-end differentiable. The package includes examples of several low- and mid-level visual models, as well as a set of perceptual quality metrics. Plenoptic is open source, tested, documented, and extensible, allowing the broader research community to contribute new examples and methods. In summary, Plenoptic leverages machine learning tools to tighten the scientific hypothesis-testing loop, facilitating investigation of human visual representations. diff --git a/joss_paper/references.bib b/joss_paper/references.bib index 4ac595ed..e8043afd 100644 --- a/joss_paper/references.bib +++ b/joss_paper/references.bib @@ -1,12 +1,12 @@ @inproceedings{simoncelli_steerable_1995, + author = {{Simoncelli}, Eero P. and {Freeman}, William T.}, title = {The steerable pyramid: a flexible architecture for multi-scale derivative computation}, volume = {3}, isbn = {0-7803-3122-2}, doi = {10.1109/ICIP.1995.537667}, booktitle = {Proceedings., International Conference on Image Processing}, publisher = {IEEE Comput. Soc. Press}, - author = {Simoncelli, Eero P. and Freeman, William T.}, year = {1995}, pages = {444--447}, } @@ -30,6 +30,7 @@ @article{berardino_eigen-distortions_2017 } @article{freeman_metamers_2011, + author = {{Freeman}, Jeremy and {Simoncelli}, Eero P.}, title = {Metamers of the ventral stream}, volume = {14}, issn = {1097-6256, 1546-1726}, @@ -38,21 +39,21 @@ @article{freeman_metamers_2011 number = {9}, urldate = {2019-09-25}, journal = {Nature Neuroscience}, - author = {Freeman, Jeremy and Simoncelli, Eero P.}, month = sep, year = {2011}, pages = {1195--1201}, } @article{paszke_pytorch_2019, + author = {{Paszke}, Adam and {Gross}, Sam and {Massa}, Francisco and {Lerer}, Adam and {Bradbury}, James and {Chanan}, Gregory and {Killeen}, Trevor and {Lin}, Zeming and {Gimelshein}, Natalia} title = {PyTorch: An Imperative Style, High-Performance Deep Learning Library}, language = {en}, - author = {Paszke, Adam and Gross, Sam and Massa, Francisco and Lerer, Adam and Bradbury, James and Chanan, Gregory and Killeen, Trevor and Lin, Zeming and Gimelshein, Natalia}, year = {2019}, pages = {12}, } @article{wang_maximum_2008, + author = {{Wang}, Zhou and {Simoncelli}, Eero P.}, title = {Maximum differentiation (MAD) competition: A methodology for comparing computational models of perceptual quantities}, volume = {8}, issn = {1534-7362}, @@ -61,17 +62,16 @@ @article{wang_maximum_2008 language = {en}, number = {12}, journal = {Journal of Vision}, - author = {Wang, Zhou and Simoncelli, Eero P.}, month = sep, year = {2008}, pages = {1-13}, } @article{portilla_parametric_2000, + author = {{Portilla}, Javier and {Simoncelli}, Eero P.}, title = {A Parametric Texture Model Based on Joint Statistics of Complex Wavelet Coefficients}, language = {en}, journal = {International Journal of Computer Vision}, year = {2000}, - author = {Portilla, Javier and Simoncelli, Eero P.}, pages = {49-71}, } From ed0d39852083d64bffb88f35a0acc4cf218bf120 Mon Sep 17 00:00:00 2001 From: Lyndon Duong Date: Tue, 10 Jan 2023 11:11:48 -0500 Subject: [PATCH 14/15] actions checkout v2 -> v3 --- .github/workflows/draft-pdf.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/draft-pdf.yml b/.github/workflows/draft-pdf.yml index 7c088ba7..3823985e 100644 --- a/.github/workflows/draft-pdf.yml +++ b/.github/workflows/draft-pdf.yml @@ -13,7 +13,7 @@ jobs: name: Paper Draft steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Build draft PDF uses: openjournals/openjournals-draft-action@master with: From 07bcd2c94d6630e816ada0f752141e7d4d3c5faf Mon Sep 17 00:00:00 2001 From: Lyndon Duong Date: Tue, 10 Jan 2023 12:02:23 -0500 Subject: [PATCH 15/15] revert --- .github/workflows/draft-pdf.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/draft-pdf.yml b/.github/workflows/draft-pdf.yml index 3823985e..7c088ba7 100644 --- a/.github/workflows/draft-pdf.yml +++ b/.github/workflows/draft-pdf.yml @@ -13,7 +13,7 @@ jobs: name: Paper Draft steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v2 - name: Build draft PDF uses: openjournals/openjournals-draft-action@master with: