Skip to content

Commit c713ba5

Browse files
5had3z5had3z
authored
Add Debayer CPU based on OpenCV (NVIDIA#5832)
* Added CPU impl of debayering w/ cv::cvtColor. Added CPU device tests. Unify OpenCV utility functions (+clang-format). * Use cv::demosaicing directly. * Use Edge-Aware algorithm for better image quality. Use an approximate equality for cpu algorithm. * Make algorithm required argument. Add opencv algo variants. Update python docs. Update tests with required algo arg. * Add tests for opencv cpu algos. Bugfix output channel for grey decode. Add assert and note for vng only supports uint8. * Remove algorithm arg requirement. Replace `bilinear_npp` with `default_npp`. Add test to check error raised if bad algo arg given depending on input device. * Update opencv-python dependency to match DALI_deps version. --------- Signed-off-by: Bryce Ferenczi <[email protected]>
1 parent 3559eb2 commit c713ba5

File tree

15 files changed

+476
-151
lines changed

15 files changed

+476
-151
lines changed

‎dali/kernels/imgproc/color_manipulation/debayer/debayer.h

Lines changed: 35 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,12 @@ enum class DALIBayerPattern {
3333
};
3434

3535
enum class DALIDebayerAlgorithm {
36-
DALI_DEBAYER_BILINEAR_NPP = 0
36+
DALI_DEBAYER_DEFAULT = 0,
37+
DALI_DEBAYER_DEFAULT_NPP = 1,
38+
DALI_DEBAYER_BILINEAR_OCV = 2,
39+
DALI_DEBAYER_EDGEAWARE_OCV = 3,
40+
DALI_DEBAYER_VNG_OCV = 4,
41+
DALI_DEBAYER_GRAY_OCV = 5,
3742
};
3843

3944
inline std::string to_string(DALIBayerPattern bayer_pattern) {
@@ -53,17 +58,42 @@ inline std::string to_string(DALIBayerPattern bayer_pattern) {
5358

5459
inline std::string to_string(DALIDebayerAlgorithm alg) {
5560
switch (alg) {
56-
case DALIDebayerAlgorithm::DALI_DEBAYER_BILINEAR_NPP:
57-
return "bilinear_npp";
61+
case DALIDebayerAlgorithm::DALI_DEBAYER_DEFAULT:
62+
return "default";
63+
case DALIDebayerAlgorithm::DALI_DEBAYER_DEFAULT_NPP:
64+
return "default_npp";
65+
case DALIDebayerAlgorithm::DALI_DEBAYER_BILINEAR_OCV:
66+
return "bilinear_ocv";
67+
case DALIDebayerAlgorithm::DALI_DEBAYER_EDGEAWARE_OCV:
68+
return "edgeaware_ocv";
69+
case DALIDebayerAlgorithm::DALI_DEBAYER_VNG_OCV:
70+
return "vng_ocv";
71+
case DALIDebayerAlgorithm::DALI_DEBAYER_GRAY_OCV:
72+
return "gray_ocv";
5873
default:
5974
return "<unknown>";
6075
}
6176
}
6277

6378
inline DALIDebayerAlgorithm parse_algorithm_name(std::string alg) {
6479
std::transform(alg.begin(), alg.end(), alg.begin(), [](auto c) { return std::tolower(c); });
65-
if (alg == "bilinear_npp") {
66-
return DALIDebayerAlgorithm::DALI_DEBAYER_BILINEAR_NPP;
80+
if (alg == "default") {
81+
return DALIDebayerAlgorithm::DALI_DEBAYER_DEFAULT;
82+
}
83+
if (alg == "default_npp") {
84+
return DALIDebayerAlgorithm::DALI_DEBAYER_DEFAULT_NPP;
85+
}
86+
if (alg == "bilinear_ocv") {
87+
return DALIDebayerAlgorithm::DALI_DEBAYER_BILINEAR_OCV;
88+
}
89+
if (alg == "edgeaware_ocv") {
90+
return DALIDebayerAlgorithm::DALI_DEBAYER_EDGEAWARE_OCV;
91+
}
92+
if (alg == "vng_ocv") {
93+
return DALIDebayerAlgorithm::DALI_DEBAYER_VNG_OCV;
94+
}
95+
if (alg == "gray_ocv") {
96+
return DALIDebayerAlgorithm::DALI_DEBAYER_GRAY_OCV;
6797
}
6898
throw std::runtime_error(
6999
make_string("Unsupported debayer algorithm was specified: `", alg, "`."));

‎dali/kernels/imgproc/color_manipulation/debayer/debayer_test.cc

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ class DebayerGpuTest : public ::testing::Test {
4646
using InOutT = typename DebayerTestParamsT::InOutT;
4747
using Kernel = NppDebayerKernel<InOutT>;
4848
static constexpr int num_channels = 3;
49-
static_assert(DebayerTestParamsT::alg == DALIDebayerAlgorithm::DALI_DEBAYER_BILINEAR_NPP);
49+
static_assert(DebayerTestParamsT::alg == DALIDebayerAlgorithm::DALI_DEBAYER_DEFAULT_NPP);
5050

5151
void FillWithGradient(TensorListView<StorageCPU, InOutT, 3> rgb_batch) {
5252
int max_val = std::numeric_limits<InOutT>::max();
@@ -151,8 +151,8 @@ class DebayerGpuTest : public ::testing::Test {
151151
};
152152

153153
using TestParams =
154-
::testing::Types<DebayerTestParams<uint8_t, DALIDebayerAlgorithm::DALI_DEBAYER_BILINEAR_NPP>,
155-
DebayerTestParams<uint16_t, DALIDebayerAlgorithm::DALI_DEBAYER_BILINEAR_NPP>>;
154+
::testing::Types<DebayerTestParams<uint8_t, DALIDebayerAlgorithm::DALI_DEBAYER_DEFAULT_NPP>,
155+
DebayerTestParams<uint16_t, DALIDebayerAlgorithm::DALI_DEBAYER_DEFAULT_NPP>>;
156156

157157
TYPED_TEST_SUITE(DebayerGpuTest, TestParams);
158158

‎dali/operators/image/color/debayer.cc

Lines changed: 15 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ Converts single-channel image to RGB using specified color filter array.
2424
The supported input types are ``uint8_t`` and ``uint16_t``.
2525
The input images must be 2D tensors (``HW``) or 3D tensors (``HWC``) where the number of channels is 1.
2626
The operator supports sequence of images/video-like inputs (layout ``FHW``).
27+
The output of the operator is always ``HWC`` (or ``FHWC`` for sequences).
2728
2829
For example, the following snippet presents debayering of batch of image sequences::
2930
@@ -48,7 +49,7 @@ For example, the following snippet presents debayering of batch of image sequenc
4849
source=bayered_sequence, batch=False, num_outputs=2,
4950
layout=["FHW", None]) # note the "FHW" layout, for plain images it would be "HW"
5051
debayered_sequences = fn.experimental.debayer(
51-
bayered_sequences.gpu(), blue_position=blue_positions)
52+
bayered_sequences.gpu(), blue_position=blue_positions, algorithm='default_npp')
5253
return debayered_sequences
5354
5455
)code")
@@ -94,15 +95,22 @@ For example, the ``(0, 0)``/``BG``/``BGGR`` corresponds to the following matrix
9495
- G
9596
)code",
9697
DALI_INT_VEC, true, true)
97-
.AddOptionalArg(
98+
.AddOptionalArg<std::string>(
9899
debayer::kAlgArgName,
99100
R"code(The algorithm to be used when inferring missing colours for any given pixel.
100-
Currently only ``bilinear_npp`` is supported.
101+
Different algorithms are supported on the GPU and CPU.
101102
102-
* The ``bilinear_npp`` algorithm uses bilinear interpolation to infer red and blue values.
103-
For green values a bilinear interpolation with chroma correlation is used as explained in
104-
`NPP documentation <https://docs.nvidia.com/cuda/npp/group__image__color__debayer.html>`_.)code",
105-
"bilinear_npp")
103+
**GPU Algorithms:**
104+
105+
- ``default_npp`` - default - bilinear interpolation with chroma correlation for green values.
106+
107+
**CPU Algorithms:**
108+
109+
- ``bilinear_ocv`` - default - bilinear interpolation.
110+
- ``edgeaware_ocv`` edge-aware interpolation.
111+
- ``vng_ocv`` Variable Number of Gradients (VNG) interpolation (only ``uint8_t`` supported).
112+
- ``gray_ocv`` converts the image to grayscale with bilinear interpolation.)code",
113+
nullptr)
106114
.InputLayout(0, {"HW", "HWC", "FHW", "FHWC"})
107115
.AllowSequences();
108116

‎dali/operators/image/color/debayer.h

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,8 @@ constexpr static const char *kAlgArgName = "algorithm";
3737
constexpr static const char *kBluePosArgName = "blue_position";
3838

3939
template <int ndim>
40-
TensorListShape<3> infer_output_shape(const TensorListShape<ndim> &input_shapes) {
40+
TensorListShape<3> infer_output_shape(const TensorListShape<ndim> &input_shapes,
41+
DALIDebayerAlgorithm alg) {
4142
int sample_dim = input_shapes.sample_dim();
4243
int batch_size = input_shapes.num_samples();
4344
DALI_ENFORCE(
@@ -66,7 +67,8 @@ TensorListShape<3> infer_output_shape(const TensorListShape<ndim> &input_shapes)
6667
make_string("The height and width of the image to debayer must be even. However, "
6768
"the sample at idx ",
6869
sample_idx, " has shape ", input_shapes[sample_idx], "."));
69-
TensorShape<3> out_shape{height, width, 3};
70+
const auto out_channels = alg == DALIDebayerAlgorithm::DALI_DEBAYER_GRAY_OCV ? 1 : 3;
71+
TensorShape<3> out_shape{height, width, out_channels};
7072
out_shapes.set_tensor_shape(sample_idx, out_shape);
7173
}
7274
return out_shapes;
@@ -103,9 +105,10 @@ template <typename Backend>
103105
class Debayer : public SequenceOperator<Backend, StatelessOperator> {
104106
public:
105107
using Base = SequenceOperator<Backend, StatelessOperator>;
106-
explicit Debayer(const OpSpec &spec)
107-
: Base(spec),
108-
alg_{debayer::parse_algorithm_name(spec.GetArgument<std::string>(debayer::kAlgArgName))} {
108+
explicit Debayer(const OpSpec &spec) : Base(spec) {
109+
if (spec_.HasArgument(debayer::kAlgArgName)) {
110+
alg_ = debayer::parse_algorithm_name(spec.GetArgument<std::string>(debayer::kAlgArgName));
111+
}
109112
if (!spec_.HasTensorArgument(debayer::kBluePosArgName)) {
110113
std::vector<int> blue_pos;
111114
GetSingleOrRepeatedArg(spec_, blue_pos, debayer::kBluePosArgName, 2);
@@ -122,7 +125,7 @@ class Debayer : public SequenceOperator<Backend, StatelessOperator> {
122125
output_desc.resize(1);
123126
const auto &input_shape = ws.GetInputShape(0);
124127
output_desc[0].type = ws.GetInputDataType(0);
125-
output_desc[0].shape = debayer::infer_output_shape(input_shape);
128+
output_desc[0].shape = debayer::infer_output_shape(input_shape, alg_);
126129
AcquirePatternArgument(ws, input_shape.num_samples());
127130
return true;
128131
}
@@ -144,7 +147,7 @@ class Debayer : public SequenceOperator<Backend, StatelessOperator> {
144147
}
145148

146149
debayer::DALIBayerPattern static_pattern_ = {};
147-
debayer::DALIDebayerAlgorithm alg_;
150+
debayer::DALIDebayerAlgorithm alg_ = debayer::DALIDebayerAlgorithm::DALI_DEBAYER_DEFAULT;
148151
std::vector<debayer::DALIBayerPattern> pattern_;
149152
};
150153

‎dali/operators/image/color/debayer_cpu.cc

Lines changed: 139 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,139 @@
1+
// Copyright (c) 2022-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
#include <opencv2/imgproc.hpp>
16+
17+
#include "dali/kernels/imgproc/color_manipulation/debayer/debayer.h"
18+
#include "dali/operators/image/color/debayer.h"
19+
#include "dali/util/ocv.h"
20+
21+
namespace dali {
22+
23+
using DALIBayerPattern = dali::kernels::debayer::DALIBayerPattern;
24+
25+
using DALIDebayerAlgorithm = dali::kernels::debayer::DALIDebayerAlgorithm;
26+
27+
cv::ColorConversionCodes toOpenCVColorConversionCode(DALIBayerPattern pattern,
28+
DALIDebayerAlgorithm algo) {
29+
switch (algo) {
30+
// OpenCV Bilinear
31+
case DALIDebayerAlgorithm::DALI_DEBAYER_BILINEAR_OCV:
32+
switch (pattern) {
33+
case DALIBayerPattern::DALI_BAYER_BG:
34+
return cv::COLOR_BayerBG2RGB;
35+
case DALIBayerPattern::DALI_BAYER_GB:
36+
return cv::COLOR_BayerGB2RGB;
37+
case DALIBayerPattern::DALI_BAYER_GR:
38+
return cv::COLOR_BayerGR2RGB;
39+
case DALIBayerPattern::DALI_BAYER_RG:
40+
return cv::COLOR_BayerRG2RGB;
41+
default:
42+
break;
43+
}
44+
// OpenCV Edge-aware
45+
case DALIDebayerAlgorithm::DALI_DEBAYER_EDGEAWARE_OCV:
46+
switch (pattern) {
47+
case DALIBayerPattern::DALI_BAYER_BG:
48+
return cv::COLOR_BayerBG2RGB_EA;
49+
case DALIBayerPattern::DALI_BAYER_GB:
50+
return cv::COLOR_BayerGB2RGB_EA;
51+
case DALIBayerPattern::DALI_BAYER_GR:
52+
return cv::COLOR_BayerGR2RGB_EA;
53+
case DALIBayerPattern::DALI_BAYER_RG:
54+
return cv::COLOR_BayerRG2RGB_EA;
55+
default:
56+
break;
57+
}
58+
// OpenCV VNG
59+
case DALIDebayerAlgorithm::DALI_DEBAYER_VNG_OCV:
60+
switch (pattern) {
61+
case DALIBayerPattern::DALI_BAYER_BG:
62+
return cv::COLOR_BayerBG2RGB_VNG;
63+
case DALIBayerPattern::DALI_BAYER_GB:
64+
return cv::COLOR_BayerGB2RGB_VNG;
65+
case DALIBayerPattern::DALI_BAYER_GR:
66+
return cv::COLOR_BayerGR2RGB_VNG;
67+
case DALIBayerPattern::DALI_BAYER_RG:
68+
return cv::COLOR_BayerRG2RGB_VNG;
69+
default:
70+
break;
71+
}
72+
// OpenCV Gray
73+
case DALIDebayerAlgorithm::DALI_DEBAYER_GRAY_OCV:
74+
switch (pattern) {
75+
case DALIBayerPattern::DALI_BAYER_BG:
76+
return cv::COLOR_BayerBG2GRAY;
77+
case DALIBayerPattern::DALI_BAYER_GB:
78+
return cv::COLOR_BayerGB2GRAY;
79+
case DALIBayerPattern::DALI_BAYER_GR:
80+
return cv::COLOR_BayerGR2GRAY;
81+
case DALIBayerPattern::DALI_BAYER_RG:
82+
return cv::COLOR_BayerRG2GRAY;
83+
default:
84+
break;
85+
}
86+
default:
87+
break;
88+
}
89+
DALI_FAIL("Unsupported bayer pattern code " + to_string(pattern) + "and algorithm code " +
90+
to_string(algo) + " combination for OpenCV debayering.");
91+
}
92+
93+
class DebayerCPU : public Debayer<CPUBackend> {
94+
public:
95+
explicit DebayerCPU(const OpSpec &spec) : Debayer<CPUBackend>(spec) {}
96+
97+
bool SetupImpl(std::vector<OutputDesc> &output_desc, const Workspace &ws) override {
98+
// If the algorithm is set to default, use bilinear ocv
99+
if (alg_ == debayer::DALIDebayerAlgorithm::DALI_DEBAYER_DEFAULT) {
100+
alg_ = debayer::DALIDebayerAlgorithm::DALI_DEBAYER_BILINEAR_OCV;
101+
}
102+
DALI_ENFORCE(alg_ != debayer::DALIDebayerAlgorithm::DALI_DEBAYER_DEFAULT_NPP,
103+
"default_npp algorithm is not supported on CPU.");
104+
if (alg_ == debayer::DALIDebayerAlgorithm::DALI_DEBAYER_VNG_OCV) {
105+
DALI_ENFORCE(ws.Input<CPUBackend>(0).type() == DALI_UINT8,
106+
"VNG debayering only supported with UINT8.");
107+
}
108+
return Debayer<CPUBackend>::SetupImpl(output_desc, ws);
109+
}
110+
111+
void RunImpl(Workspace &ws) override {
112+
const auto &input = ws.Input<CPUBackend>(0);
113+
auto &output = ws.Output<CPUBackend>(0);
114+
output.SetLayout("HWC");
115+
116+
auto &tPool = ws.GetThreadPool();
117+
for (int i = 0; i < input.num_samples(); ++i) {
118+
tPool.AddWork([&, i](int) {
119+
const auto inImage = input[i];
120+
auto outImage = output[i];
121+
122+
const auto &oShape = outImage.shape();
123+
const auto height = static_cast<int>(oShape[0]);
124+
const auto width = static_cast<int>(oShape[1]);
125+
const auto outChannels = static_cast<int>(oShape[2]);
126+
cv::Mat inImg(height, width, OCVMatTypeForDALIData(inImage.type(), 1),
127+
const_cast<void *>(inImage.raw_data()));
128+
cv::Mat outImg(height, width, OCVMatTypeForDALIData(outImage.type(), outChannels),
129+
outImage.raw_mutable_data());
130+
cv::demosaicing(inImg, outImg, toOpenCVColorConversionCode(pattern_[i], alg_));
131+
});
132+
}
133+
tPool.RunAll();
134+
}
135+
};
136+
137+
DALI_REGISTER_OPERATOR(experimental__Debayer, DebayerCPU, CPU);
138+
139+
} // namespace dali

‎dali/operators/image/color/debayer_gpu.cc

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -84,9 +84,14 @@ class DebayerGPU : public Debayer<GPUBackend> {
8484

8585
bool DebayerGPU::SetupImpl(std::vector<OutputDesc> &output_desc, const Workspace &ws) {
8686
bool has_inferred = Debayer<GPUBackend>::SetupImpl(output_desc, ws);
87+
// If the algorithm is set to default, use default npp
88+
if (alg_ == debayer::DALIDebayerAlgorithm::DALI_DEBAYER_DEFAULT) {
89+
alg_ = debayer::DALIDebayerAlgorithm::DALI_DEBAYER_DEFAULT_NPP;
90+
}
8791
assert(has_inferred);
8892
if (impl_ == nullptr) {
89-
assert(alg_ == debayer::DALIDebayerAlgorithm::DALI_DEBAYER_BILINEAR_NPP);
93+
DALI_ENFORCE(alg_ == debayer::DALIDebayerAlgorithm::DALI_DEBAYER_DEFAULT_NPP,
94+
"Only default and default_npp algorithm is supported on GPU.");
9095
const auto type = ws.GetInputDataType(0);
9196
TYPE_SWITCH(type, type2id, InT, DEBAYER_SUPPORTED_TYPES_GPU, (
9297
impl_ = std::make_unique<debayer::DebayerImplGPU<InT>>(spec_, pattern_);

‎dali/operators/image/remap/cvcuda/warp_perspective.cc

Lines changed: 2 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
#include "dali/pipeline/operator/arg_helper.h"
2626
#include "dali/pipeline/operator/checkpointing/stateless_operator.h"
2727
#include "dali/pipeline/operator/operator.h"
28+
#include "dali/util/ocv.h"
2829

2930
#include "dali/operators/image/remap/cvcuda/matrix_adjust.h"
3031
#include "dali/operators/nvcvop/nvcvop.h"
@@ -349,31 +350,6 @@ class WarpPerspectiveCPU : public SequenceOperator<CPUBackend, StatelessOperator
349350
matrix = shiftBack * (matrix * shift);
350351
}
351352

352-
/**
353-
* @brief Convert DALI data type to OpenCV matrix type
354-
*/
355-
int matTypeFromDALI(DALIDataType dtype) {
356-
switch (dtype) {
357-
case DALI_UINT8:
358-
return CV_8U;
359-
case DALI_INT16:
360-
return CV_16S;
361-
case DALI_UINT16:
362-
return CV_16U;
363-
case DALI_FLOAT:
364-
return CV_32F;
365-
default:
366-
DALI_FAIL("Unsupported input type");
367-
}
368-
}
369-
370-
/**
371-
* @brief Construct a full OpenCV matrix type from DALI data type and number of channels
372-
*/
373-
int fullMatTypeFromDALI(DALIDataType dtype, int channels) {
374-
return CV_MAKETYPE(matTypeFromDALI(dtype), channels);
375-
}
376-
377353
void RunImpl(Workspace &ws) override {
378354
const auto &input = ws.Input<CPUBackend>(0);
379355
auto &output = ws.Output<CPUBackend>(0);
@@ -414,7 +390,7 @@ class WarpPerspectiveCPU : public SequenceOperator<CPUBackend, StatelessOperator
414390
tPool.AddWork([&, i](int) {
415391
const auto inImage = input[i];
416392
auto outImage = output[i];
417-
const int dtype = fullMatTypeFromDALI(inImage.type(), channels_);
393+
const int dtype = OCVMatTypeForDALIData(inImage.type(), channels_);
418394

419395
const auto &inShape = inImage.shape();
420396
const cv::Mat inMat(static_cast<int>(inShape[0]), static_cast<int>(inShape[1]), dtype,

0 commit comments

Comments
 (0)