diff --git a/blackbox_suite_test.go b/blackbox_suite_test.go
new file mode 100644
index 0000000..8bc4920
--- /dev/null
+++ b/blackbox_suite_test.go
@@ -0,0 +1,13 @@
+package blackbox_test
+
+import (
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"testing"
+)
+
+func TestBlackbox(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Blackbox Suite")
+}
diff --git a/config.go b/config.go
index e3cf946..5c9eed4 100644
--- a/config.go
+++ b/config.go
@@ -1,7 +1,7 @@
 package blackbox
 
 import (
-	"fmt"
+	"errors"
 	"io/ioutil"
 	"os"
 	"time"
@@ -12,15 +12,16 @@ import (
 
 type Duration time.Duration
 
-func (d *Duration) UnmarshalYAML(tag string, value interface{}) error {
-	if num, ok := value.(int64); ok {
+func (d *Duration) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	var num int64
+	if err := unmarshal(&num); err == nil {
 		*d = Duration(num)
 		return nil
 	}
 
-	str, ok := value.(string)
-	if !ok {
-		return fmt.Errorf("invalid duration: %T (%#v)", value, value)
+	var str string
+	if err := unmarshal(&str); err != nil {
+		return errors.New("invalid duration; must be string or number")
 	}
 
 	duration, err := time.ParseDuration(str)
diff --git a/config_test.go b/config_test.go
new file mode 100644
index 0000000..e1fe629
--- /dev/null
+++ b/config_test.go
@@ -0,0 +1,31 @@
+package blackbox_test
+
+import (
+	"time"
+
+	. "github.com/concourse/blackbox"
+	"gopkg.in/yaml.v2"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Config", func() {
+	Describe("Duration", func() {
+		It("can be unmarshalled from YAML as a string", func() {
+			var duration Duration
+			err := yaml.Unmarshal([]byte("10s"), &duration)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(time.Duration(duration)).To(Equal(10 * time.Second))
+		})
+
+		It("can be unmarshalled from YAML as an integer", func() {
+			var duration Duration
+			err := yaml.Unmarshal([]byte("10"), &duration)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(time.Duration(duration)).To(Equal(10 * time.Nanosecond))
+		})
+	})
+})