Skip to content

DeepLab v3 does not work with High Level API of Python  #19481

@UnaNancyOwen

Description

@UnaNancyOwen
System information (version)
  • OpenCV => 4.5.1
  • Operating System / Platform => Windows 10 x64
  • Compiler => Visual Studio 2019
  • Python => 3.8
Detailed description

Python High Level API (cv.dnn_SegmentationModel) is not able to inference DeepLab v3 model.
However, It is able to inference with Python Low Level API (cv.dnn.Net) and C++ High Level API (cv::dnn::SegmentationModel) and C++ Low Level API (cv::dnn::Net).

Steps to reproduce
  • Python High Level API (Not Work)
import numpy as np
import cv2 as cv

# Read Classes Name
def read_class_name_list(file):
    classes = None
    with open(file, 'r') as f:
        classes = f.read().rstrip('\n').split('\n')
    return classes

# Get Classes Color
def get_class_colors(num):
    colors = []
    np.random.seed(0)
    for i in range(num):
        color = np.random.randint(0, 256, [3]).tolist()
        colors.append(color)
    return colors

if __name__ == '__main__':
    # Open Capture
    capture = cv.VideoCapture("bicycle.jpg")
    if not capture.isOpened():
        exit()

    # Create Model
    weights = "optimized_graph.pb"
    model = cv.dnn_SegmentationModel(weights)

    # Normalize Parameters
    scale = 0.007843
    size = (513, 513)
    mean = (127.5, 127.5, 127.5)
    swap = True
    crop = False
    model.setInputParams(scale, size, mean, swap, crop)

    # Read Classes Name and Colors
    names = "voc.names"
    classes = read_class_name_list(names)
    colors = get_class_colors(len(classes))

    while True:
        # Capture Frame
        result, image = capture.read()
        if result is False:
            cv.waitKey(0)
            break

        # Segmentation
        mask = model.segment(image) ## crash!!! ##

        # Generate Color Mask
        rows, cols = mask.shape
        color_mask = np.full(np.array([rows, cols, 3]), np.full(3, 0.), np.uint8)
        for y in range(rows):
            for x in range(cols):
                if mask[y, x] != 0: color_mask[y, x] = colors[mask[y, x]]
        color_mask = cv.resize(color_mask, (image.shape[1], image.shape[0]), cv.INTER_NEAREST)

        # Alpha Blend
        alpha = 0.5
        beta  = 1.0 - alpha
        cv.addWeighted(image, alpha, color_mask, beta, 0.0, image)

        # Show Image
        cv.imshow("segmentation", image)
        cv.imshow("mask", color_mask)
        key = cv.waitKey(10)
        if key == ord('q'):
            break
    
    cv.destroyAllWindows()
  • C++ High Level API (Work Fine)
#include <iostream>
#include <string>
#include <vector>
#include <fstream>
#include <type_traits>
#include <opencv2/opencv.hpp>

// Read Class Name List
std::vector<std::string> readClassNameList( const std::string list_path )
{
    std::vector<std::string> classes;
    std::ifstream ifs( list_path );
    if( !ifs.is_open() ){
        return classes;
    }
    std::string class_name = "";
    while( std::getline( ifs, class_name ) ){
        classes.push_back( class_name );
    }
    return classes;
}

// Get Class Color Table for Visualize
std::vector<cv::Scalar> getClassColors( const int32_t number_of_colors )
{
    cv::RNG random;
    std::vector<cv::Scalar> colors;
    for( int32_t i = 0; i < number_of_colors; i++ ){
        cv::Scalar color( random.uniform( 0, 255 ), random.uniform( 0, 255 ), random.uniform( 0, 255 ) );
        colors.push_back( color );
    }
    return colors;
}

int main( int argc, char* argv[] )
{
    // Open Video Capture
    cv::VideoCapture capture = cv::VideoCapture( "bicycle.jpg" );
    if( !capture.isOpened() ){
        return -1;
    }

    // Create Model
    const std::string weights = "optimized_graph.pb";
    cv::dnn::SegmentationModel model = cv::dnn::SegmentationModel( weights );

    // Normalize Parameters
    constexpr double scale = 0.007843;
    const cv::Size size = cv::Size( 513, 513 );
    const cv::Scalar mean = cv::Scalar( 127.5, 127.5, 127.5 );
    constexpr bool swap = true;
    constexpr bool crop = false;
    model.setInputParams( scale, size, mean, swap, crop );

    // Read Classes Name and Colors
    const std::string list = "voc.names";
    const std::vector<std::string> classes = readClassNameList( list );
    const std::vector<cv::Scalar> colors = getClassColors( classes.size() );

    while( true ){
        // Capture Frame
        cv::Mat image;
        capture >> image;
        if( image.empty() ){
            cv::waitKey( 0 );
            break;
        }

        // Segmentation
        cv::Mat mask;
        model.segment( image, mask );

        // Generate Color Mask
        const int32_t rows = mask.rows;
        const int32_t cols = mask.cols;
        cv::Mat color_mask = cv::Mat::zeros( rows, cols, CV_8UC3 );
        for( int32_t y = 0; y < rows; y++ ){
            for( int32_t x = 0; x < cols; x++ ){
                const uint8_t class_id = mask.at<uint8_t>( y, x );
                if( class_id == 0 ) continue;
                const cv::Scalar color = colors[class_id];
                color_mask.at<cv::Vec3b>( y, x ) = cv::Vec3b( color[0], color[1], color[2] );
            }
        }
        cv::resize( color_mask, color_mask, image.size() );

        // Alpha Blend
        constexpr double alpha = 0.5;
        constexpr double beta = 1.0 - alpha;
        cv::addWeighted( image, alpha, color_mask, beta, 0.0, image );

        // Show Image
        cv::imshow( "segmentation", image );
        const int32_t key = cv::waitKey( 10 );
        if( key == 'q' ){
            break;
        }
    }

    cv::destroyAllWindows();

    return 0;
}
Issue submission checklist
  • I report the issue, it's not a question
  • I checked the problem with documentation, FAQ, open issues,
    forum.opencv.org, Stack Overflow, etc and have not found solution
  • I updated to latest OpenCV version and the issue is still there
  • There is reproducer code and related data files: videos, images, onnx, etc

Metadata

Metadata

Assignees

No one assigned

    Type

    No type

    Projects

    No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions