Merge pull request #6 from kengz/1.0.0

1.0.0 replace TensorTuple with tensor dict

.github/workflows/ci.yml

@@ -4,10 +4,39 @@ on:
  push:
  branches: [main]
  pull_request:
- branches: [main]
+ branches: ["**"]
 
 jobs:
+ lint:
+ runs-on: ubuntu-latest
+
+ steps:
+ - name: Check out Git repository
+ uses: actions/checkout@v2
+
+ - name: Set up Python
+ uses: actions/setup-python@v2
+ with:
+ python-version: 3.8
+
+ - uses: liskin/gh-problem-matcher-wrap@v1
+ with:
+ action: add
+ linters: flake8
+
+ - name: Lint with flake8
+ run: |
+ pip install flake8
+ # exit-zero treats all errors as warnings.
+ flake8 . --ignore=E501 --count --exit-zero --statistics
+
+ - uses: liskin/gh-problem-matcher-wrap@v1
+ with:
+ action: remove
+ linters: flake8
+
  build:
+ needs: lint
  runs-on: ubuntu-latest
 
  steps:
@@ -35,16 +64,19 @@ jobs:
  conda info
  conda list
 
- - name: Setup flake8 annotations
- uses: rbialon/flake8-annotations@v1
- - name: Lint with flake8
- shell: bash -l {0}
- run: |
- pip install flake8
- # exit-zero treats all errors as warnings.
- flake8 . --ignore=E501 --count --exit-zero --statistics
+ - uses: liskin/gh-problem-matcher-wrap@v1
+ with:
+ action: add
+ linters: pytest
 
  - name: Run tests
  shell: bash -l {0}
- run: |
- python setup.py test
+ run: python setup.py test | tee pytest-coverage.txt
+
+ - name: Post coverage to PR comment
+ uses: coroo/[email protected]
+
+ - uses: liskin/gh-problem-matcher-wrap@v1
+ with:
+ action: add
+ linters: pytest

.github/workflows/tag-release.yml

@@ -0,0 +1,30 @@
+# tag ref: https://github.com/marketplace/actions/github-tag#bumping
+# commit msg format for tag: https://github.com/angular/angular.js/blob/master/DEVELOPERS.md#-git-commit-guidelines
+name: Tag and release version
+
+on:
+ push:
+ branches: [main]
+
+jobs:
+ tag_and_release:
+ runs-on: ubuntu-latest
+
+ steps:
+ - uses: actions/checkout@v2
+
+ - name: Bump version and push tag
+ id: tag_version
+ uses: mathieudutour/[email protected]
+ with:
+ github_token: ${{ secrets.GITHUB_TOKEN }}
+ tag_prefix: ''
+
+ - name: Create a GitHub release
+ uses: actions/create-release@v1
+ env:
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ with:
+ tag_name: ${{ steps.tag_version.outputs.new_tag }}
+ release_name: Release ${{ steps.tag_version.outputs.new_tag }}
+ body: ${{ steps.tag_version.outputs.changelog }}

README.md

@@ -359,10 +359,8 @@ model = torcharc.build(arc)
 
 batch_size = 16
 dag_in_shape = arc['dag_in_shape']
-data = {'image': torch.rand([batch_size, *dag_in_shape['image']]), 'vector': torch.rand([batch_size, *dag_in_shape['vector']])}
-# convert from a dict of Tensors into a TensorTuple - a namedtuple
-xs = torcharc.to_namedtuple(data)
-# returns TensorTuple if output is multi-model, Tensor otherwise
+xs = {'image': torch.rand([batch_size, *dag_in_shape['image']]), 'vector': torch.rand([batch_size, *dag_in_shape['vector']])}
+# returns dict if output is multi-model, Tensor otherwise
 ys = model(xs)
 ```
 
@@ -406,9 +404,9 @@ DAGNet(
 </p>
 </details>
 
-DAG module accepts a `TensorTuple` (example below) as input, and the module selects its input by matching its own name in the arc and the `in_name`, then carry forward the output together with any unconsumed inputs.
+DAG module accepts a `dict` (example below) as input, and the module selects its input by matching its own name in the arc and the `in_name`, then carry forward the output together with any unconsumed inputs.
 
-For example, the input `xs` with keys `image, vector` passes through the first `image` module, and the output becomes `TensorTuple(image=image_module(xs.image), vector=xs.vector)`. This is then passed through the remainder of the modules in the arc as declared.
+For example, the input `xs` with keys `image, vector` passes through the first `image` module, and the output becomes `{'image': image_module(xs.image), 'vector': xs.vector}`. This is then passed through the remainder of the modules in the arc as declared.
 
 ## Development
 

setup.py

@@ -11,9 +11,6 @@
  '--log-file-level=INFO',
  '--no-flaky-report',
  '--timeout=300',
- '--cov-report=html',
- '--cov-report=term',
- '--cov-report=xml',
  '--cov=torcharc',
  'test',
 ]
@@ -35,7 +32,7 @@ def run_tests(self):
 
 setup(
  name='torcharc',
- version='0.0.6',
+ version='1.0.0',
  description='Build PyTorch networks by specifying architectures.',
  long_description='https://github.com/kengz/torcharc',
  keywords='torcharc',

test/module/test_dag.py

@@ -1,6 +1,5 @@
 from torcharc import arc_ref, net_util
 from torcharc.module import dag
-import pydash as ps
 import torch
 
 
@@ -37,7 +36,7 @@ def test_dag_reusefork():
  xs = net_util.get_rand_tensor(in_shapes)
  model = dag.DAGNet(arc)
  ys = model(xs)
- assert ps.is_tuple(ys)
+ assert isinstance(ys, dict)
 
 
 def test_dag_splitfork():
@@ -46,17 +45,16 @@ def test_dag_splitfork():
  xs = net_util.get_rand_tensor(in_shapes)
  model = dag.DAGNet(arc)
  ys = model(xs)
- assert ps.is_tuple(ys)
+ assert isinstance(ys, dict)
 
 
 def test_dag_merge_fork():
  arc = arc_ref.REF_ARCS['merge_fork']
  in_shapes = arc['dag_in_shape']
  xs = net_util.get_rand_tensor(in_shapes)
  model = dag.DAGNet(arc)
- ys = model(xs._asdict()) # test dict input for tracing
  ys = model(xs)
- assert ps.is_tuple(ys)
+ assert isinstance(ys, dict)
 
 
 def test_dag_fork_merge():
@@ -74,7 +72,7 @@ def test_dag_reuse_fork_forward():
  xs = net_util.get_rand_tensor(in_shapes)
  model = dag.DAGNet(arc)
  ys = model(xs)
- assert ps.is_tuple(ys)
+ assert isinstance(ys, dict)
 
 
 def test_dag_split_fork_forward():
@@ -83,24 +81,22 @@ def test_dag_split_fork_forward():
  xs = net_util.get_rand_tensor(in_shapes)
  model = dag.DAGNet(arc)
  ys = model(xs)
- assert ps.is_tuple(ys)
+ assert isinstance(ys, dict)
 
 
 def test_dag_merge_forward_split():
  arc = arc_ref.REF_ARCS['merge_forward_split']
  in_shapes = arc['dag_in_shape']
  xs = net_util.get_rand_tensor(in_shapes)
  model = dag.DAGNet(arc)
- ys = model(xs._asdict()) # test dict input for tracing
  ys = model(xs)
- assert ps.is_tuple(ys)
+ assert isinstance(ys, dict)
 
 
 def test_dag_hydra():
  arc = arc_ref.REF_ARCS['hydra']
  in_shapes = arc['dag_in_shape']
  xs = net_util.get_rand_tensor(in_shapes)
  model = dag.DAGNet(arc)
- ys = model(xs._asdict()) # test dict input for tracing
  ys = model(xs)
- assert ps.is_tuple(ys)
+ assert isinstance(ys, dict)

test/module/test_fork.py

@@ -1,5 +1,4 @@
 from torcharc.module.fork import Fork, ReuseFork, SplitFork
-import pydash as ps
 import pytest
 import torch
 
@@ -14,8 +13,8 @@ def test_reuse_fork(names, x):
  fork = ReuseFork(names)
  assert isinstance(fork, Fork)
  ys = fork(x)
- assert ps.is_tuple(ys)
- assert ys._fields == tuple(names)
+ assert isinstance(ys, dict)
+ assert list(ys) == names
 
 
 @pytest.mark.parametrize('shapes,x', [
@@ -28,5 +27,5 @@ def test_split_fork(shapes, x):
  fork = SplitFork(shapes)
  assert isinstance(fork, Fork)
  ys = fork(x)
- assert ps.is_tuple(ys)
- assert ys._fields == tuple(shapes.keys())
+ assert isinstance(ys, dict)
+ assert ys.keys() == shapes.keys()

test/module/test_merge.py

@@ -16,7 +16,7 @@
 def test_concat_merge(xs, out_shape):
  merge = ConcatMerge()
  assert isinstance(merge, Merge)
- y = merge(net_util.to_namedtuple(xs))
+ y = merge(xs)
  assert y.shape == torch.Size(out_shape)
 
 
@@ -56,5 +56,5 @@ def test_film_affine_transform(feature):
 def test_film_merge(names, shapes, xs):
  merge = FiLMMerge(names, shapes)
  assert isinstance(merge, Merge)
- y = merge(net_util.to_namedtuple(xs))
+ y = merge(xs)
  assert y.shape == xs[names['feature']].shape

test/test_module_builder.py

@@ -1,7 +1,6 @@
 from fixture.net import CONV1D_ARC, CONV2D_ARC, CONV3D_ARC, LINEAR_ARC
 from torcharc import module_builder, net_util
 from torch import nn
-import pydash as ps
 import pytest
 import torch
 
@@ -27,7 +26,7 @@
 ])
 def test_get_init_fn(init, activation):
  init_fn = module_builder.get_init_fn(init, activation)
- assert ps.is_function(init_fn)
+ assert callable(init_fn)
 
 
 @pytest.mark.parametrize('arc,nn_class', [
@@ -248,11 +247,11 @@ def test_carry_forward_tensor(arc, xs):
  net_util.get_rand_tensor({'vector': [LINEAR_ARC['in_features']], 'image': CONV2D_ARC['in_shape']}),
  )
 ])
-def test_carry_forward_tensor_tuple_default(arc, xs):
+def test_carry_forward_dict_default(arc, xs):
  module = module_builder.build_module(arc)
- assert ps.is_tuple(xs)
+ assert isinstance(xs, dict)
  ys = module_builder.carry_forward(module, xs)
- assert ps.is_tuple(ys)
+ assert isinstance(ys, dict)
 
 
 @pytest.mark.parametrize('xs', [
@@ -281,8 +280,8 @@ def test_carry_forward_tensor_tuple_default(arc, xs):
  ['image', 'vector'],
  )
 ])
-def test_carry_forward_tensor_tuple(arc, xs, in_names):
+def test_carry_forward_dict(arc, xs, in_names):
  module = module_builder.build_module(arc)
- assert ps.is_tuple(xs)
+ assert isinstance(xs, dict)
  ys = module_builder.carry_forward(module, xs, in_names)
- assert isinstance(ys, (torch.Tensor, tuple))
+ assert isinstance(ys, (torch.Tensor, dict))

test/test_net_util.py

@@ -1,7 +1,6 @@
 from fixture.net import CONV1D_ARC, CONV2D_ARC, CONV3D_ARC, LINEAR_ARC
 from torcharc import module_builder, net_util
 from torch import nn
-import pydash as ps
 import pytest
 import torch
 
@@ -39,7 +38,7 @@ def test_get_rand_tensor(shape, batch_size, tensor_shape):
 ])
 def test_get_rand_tensor_dict(shapes, batch_size, tensor_shapes):
  xs = net_util.get_rand_tensor(shapes, batch_size)
- assert ps.is_tuple(xs)
+ assert isinstance(xs, dict)
  for name, tensor_shape in tensor_shapes.items():
- x = getattr(xs, name)
+ x = xs[name]
  assert list(x.shape) == tensor_shape

torcharc/__init__.py

@@ -1,5 +1,4 @@
 from torcharc import module_builder
-from torcharc.net_util import to_namedtuple
 from torcharc.module import dag
 from torch import nn
 

torcharc/module/dag.py

@@ -1,7 +1,7 @@
 # build DAG of nn modules
 from torcharc import module_builder, net_util
 from torch import nn
-from typing import NamedTuple, Union
+from typing import Union
 import pydash as ps
 import torch
 
@@ -30,10 +30,8 @@ def __init__(self, arc: dict) -> None:
  xs = module_builder.carry_forward(module, xs, m_arc.get('in_names'))
  self.module_dict.update({name: module})
 
- def forward(self, xs: Union[torch.Tensor, NamedTuple]) -> Union[torch.Tensor, NamedTuple]:
- # jit.trace will spread args on encountering a namedtuple, thus xs needs to be passed as dict then converted back into namedtuple
- if ps.is_dict(xs): # guard to convert dict xs into namedtuple
- xs = net_util.to_namedtuple(xs)
+ def forward(self, xs: Union[torch.Tensor, dict]) -> Union[torch.Tensor, dict]:
+ # safe for jit.trace
  for name, module in self.module_dict.items():
  m_arc = self.arc[name]
  xs = module_builder.carry_forward(module, xs, m_arc.get('in_names'))

torcharc/module/fork.py

@@ -1,7 +1,6 @@
 from abc import ABC, abstractmethod
-from collections import namedtuple
 from torch import nn
-from typing import Dict, List, NamedTuple
+from typing import Dict, List
 import pydash as ps
 import torch
 
@@ -10,21 +9,20 @@ class Fork(ABC, nn.Module):
  '''A Fork module forks one tensor into a dict of multiple tensors.'''
 
  @abstractmethod
- def forward(self, x: torch.Tensor) -> NamedTuple: # pragma: no cover
+ def forward(self, x: torch.Tensor) -> dict: # pragma: no cover
  raise NotImplementedError
 
 
 class ReuseFork(Fork):
- '''Fork layer to reuse a tensor multiple times via ref in TensorTuple'''
+ '''Fork layer to reuse a tensor multiple times via ref in dict'''
 
  def __init__(self, names: List[str]) -> None:
  super().__init__()
  self.names = names
  self.num_reuse = len(names)
- self.TensorTuple = namedtuple('TensorTuple', names)
 
- def forward(self, x: torch.Tensor) -> NamedTuple:
- return self.TensorTuple(*[x] * self.num_reuse)
+ def forward(self, x: torch.Tensor) -> dict:
+ return dict(zip(self.names, [x] * self.num_reuse))
 
 
 class SplitFork(Fork):
@@ -34,7 +32,6 @@ def __init__(self, shapes: Dict[str, List[int]]) -> None:
  super().__init__()
  self.shapes = shapes
  self.split_size = ps.flatten(self.shapes.values())
- self.TensorTuple = namedtuple('TensorTuple', shapes.keys())
 
- def forward(self, x: torch.Tensor) -> NamedTuple:
- return self.TensorTuple(*x.split(self.split_size, dim=1))
+ def forward(self, x: torch.Tensor) -> dict:
+ return dict(zip(self.shapes, x.split(self.split_size, dim=1)))