I personally found that the rendering script provided by Liu is not of high quality.
Although I am not professional but I tweaked some settings to improve the quality of the rendering result.
I would like to share the code for anyone who wants to render without using blender.

The results look like this:

Here is the code.

import os
import multiprocessing
import torch as th
import imageio
import smplx
import trimesh
import pyrender
import numpy as np
import moviepy.editor as mp
import matplotlib
import matplotlib.pyplot as plt
from pyvirtualdisplay import Display


# Some configurations
resolution = (1000, 1000)
figsize = (10, 10)


def deg_to_rad(degrees):
    return degrees * np.pi / 180


def process_frame(i, vertices_all, faces, output_dir):
    matplotlib.use('Agg')

    filename = f'{output_dir}frame_{i}.png'
    if i % 100 == 0:
        print('[ info ] processed', i, 'frames')
    
    trimesh_mesh = trimesh.Trimesh(
        vertices=vertices_all[i], 
        faces=faces,
        vertex_colors=[220, 220, 220]
    )
    mesh = pyrender.Mesh.from_trimesh(
        trimesh_mesh, smooth=True
    )

    scene = pyrender.Scene()
    scene.add(mesh)

    angle_rad = deg_to_rad(-10)
    pose_camera = np.array([
        [1.0, 0.0, 0.0, 0.0],
        [0.0, np.cos(angle_rad), -np.sin(angle_rad), 0.0],
        [0.0, np.sin(angle_rad), np.cos(angle_rad), 2.0],
        [0.0, 0.0, 0.0, 1.0]
    ])
    camera = pyrender.OrthographicCamera(xmag=1.0, ymag=1.0)
    scene.add(camera, pose=pose_camera)

    angle_rad = deg_to_rad(-30)
    pose_light = np.array([
        [1.0, 0.0, 0.0, 0.0],
        [0.0, np.cos(angle_rad), -np.sin(angle_rad), 0.0],
        [0.0, np.sin(angle_rad), np.cos(angle_rad), 2.0],
        [0.0, 0.0, 0.0, 1.0]
    ])
    light = pyrender.DirectionalLight(color=[1.0, 1.0, 1.0], intensity=4.0)
    scene.add(light, pose=pose_light)

    renderer = pyrender.OffscreenRenderer(*resolution)
    color, _ = renderer.render(scene)

    fig = plt.figure(figsize=figsize)
    plt.subplot(1,1,1)
    plt.axis('off')
    plt.imshow(color)

    plt.savefig(filename, bbox_inches='tight')
    plt.close(fig)
    renderer.delete()


def generate_images(frames, vertices_all, faces, output_dir):
    num_cores = multiprocessing.cpu_count() - 1  # This will get the number of cores on your machine.
    # Use a Pool to manage the processes
    with multiprocessing.Pool(num_cores) as pool:
        pool.starmap(
            process_frame, 
            [
                (i, vertices_all, faces, output_dir) 
                for i in range(frames)
            ]
        )


def render_one_sequence(
    npz_path,
    output_dir,
    audio_path,
    output_name='output',
    model_folder='./smplx_models',
    debug=False
):
    display = Display(visible=0, size=resolution)
    display.start()
    
    model = smplx.create(
        model_folder, 
        model_type='smplx',
        gender='NEUTRAL_2020', 
        use_face_contour=False,
        num_betas=300,
        num_expression_coeffs=100,
        ext='npz', 
        use_pca=False
    ).cuda()
    
    data_np_body = np.load(npz_path, allow_pickle=True)
    
    if not os.path.exists(output_dir): 
        os.makedirs(output_dir)

    faces = np.load(f'{model_folder}/smplx/SMPLX_NEUTRAL_2020.npz', allow_pickle=True)['f']
    
    n = data_np_body['poses'].shape[0]
    # beta = th.from_numpy(data_np_body['betas']).float().unsqueeze(0).cuda()
    # beta = beta.repeat(n, 1)
    # expression = th.from_numpy(data_np_body['expressions'][:n]).float().cuda()
    betas = th.zeros(n, 300).float().cuda()
    expression = th.zeros(n, 100).float().cuda()
    jaw_pose = th.from_numpy(data_np_body['poses'][:n, 66:69]).float().cuda()
    pose = th.from_numpy(data_np_body['poses'][:n]).float().cuda()
    transl = th.from_numpy(data_np_body['trans'][:n]).float().cuda()
    output = model(
        betas=betas,
        expression=expression,
        transl=transl,
        jaw_pose=jaw_pose,
        global_orient=pose[:,:3], 
        body_pose=pose[:,3:21*3+3], 
        left_hand_pose=pose[:,25*3:40*3], 
        right_hand_pose=pose[:,40*3:55*3],
        leye_pose=pose[:, 69:72], 
        reye_pose=pose[:, 72:75],
        return_verts=True
    )
    vertices_all = output['vertices'].cpu().detach().numpy()

    if debug:
        seconds = 1
    else:
        seconds = vertices_all.shape[0]//30
    # camera_settings = None    
    # time_s = time.time()
    generate_images(int(seconds*30), vertices_all, faces, output_dir)
    filenames = [f'{output_dir}frame_{i}.png' for i in range(int(seconds*30))]

    # display.stop()
    # print(filenames)
    images = [imageio.imread(filename) for filename in filenames]
    imageio.mimsave(f'{output_dir}raw_{output_name}.mp4', images, fps=30)
    for filename in filenames:
        os.remove(filename)
        
    video = mp.VideoFileClip(f'{output_dir}raw_{output_name}.mp4')
    audio = mp.AudioFileClip(audio_path)
    if audio.duration > video.duration:
        audio = audio.subclip(0, video.duration)
    final_clip = video.set_audio(audio)
    final_clip.write_videofile(f'{output_dir}{output_name}.mp4')
    os.remove(f'{output_dir}raw_{output_name}.mp4')
    
    display.stop()
    
    
def process_frame_multi(
    i,
    vertices_all_list,
    title_list,
    faces,
    output_dir,
    max_n_cols=4
):
    matplotlib.use('Agg')

    vertices_list = [
        vertices_all[i] 
        for vertices_all in vertices_all_list
    ]
    filename = f'{output_dir}frame_{i}.png'
    if i%100 == 0:
        print('processed', i, 'frames')

    num_vtx = len(vertices_list)
    n_cols = min(num_vtx, max_n_cols)
    n_rows = int(np.ceil(num_vtx / n_cols))
    fig, axs = plt.subplots(
        nrows=n_rows, 
        ncols=n_cols, 
        figsize=(figsize[0] * n_cols, figsize[1] * n_rows)
    )
    axs = axs.flatten()

    angle_rad = deg_to_rad(-10)
    pose_camera = np.array([
        [1.0, 0.0, 0.0, 0.0],
        [0.0, np.cos(angle_rad), -np.sin(angle_rad), 0.0],
        [0.0, np.sin(angle_rad), np.cos(angle_rad), 2.0],
        [0.0, 0.0, 0.0, 1.0]
    ])
    angle_rad = deg_to_rad(-30)
    pose_light = np.array([
        [1.0, 0.0, 0.0, 0.0],
        [0.0, np.cos(angle_rad), -np.sin(angle_rad), 0.0],
        [0.0, np.sin(angle_rad), np.cos(angle_rad), 2.0],
        [0.0, 0.0, 0.0, 1.0]
    ])
    
    for vtx_idx in range(num_vtx):
        trimesh_mesh = trimesh.Trimesh(
            vertices=vertices_list[vtx_idx],
            faces=faces,
            vertex_colors=[220, 220, 220]
        )
        mesh = pyrender.Mesh.from_trimesh(
            trimesh_mesh, smooth=True
        )
        scene = pyrender.Scene()
        scene.add(mesh)
        camera = pyrender.OrthographicCamera(xmag=1.0, ymag=1.0)
        scene.add(camera, pose=pose_camera)
        light = pyrender.DirectionalLight(color=[1.0, 1.0, 1.0], intensity=4.0)
        scene.add(light, pose=pose_light)
        renderer = pyrender.OffscreenRenderer(*resolution)
        color, _ = renderer.render(scene)
        axs[vtx_idx].imshow(color)
        axs[vtx_idx].axis('off')
        axs[vtx_idx].set_title(title_list[vtx_idx])
        renderer.delete()
            
    plt.savefig(filename, bbox_inches='tight')
    plt.close(fig)


def generate_images_multi(
    frames,
    vertices_all_list,
    title_list,
    faces,
    output_dir
):
    num_cores = multiprocessing.cpu_count()  # This will get the number of cores on your machine.
    # Use a Pool to manage the processes
    with multiprocessing.Pool(num_cores) as pool:
        pool.starmap(
            process_frame_multi, 
            [
                (i, vertices_all_list, title_list, faces, output_dir) 
                for i in range(frames)
            ]
        )


def render_multiple_sequences(
    npz_path_list,
    output_dir,
    audio_path,
    title_list,
    output_name='output',
    model_folder='./modules/smplx_models',
    debug=False
):
    assert len(npz_path_list) == len(title_list), \
        'The number of npz files and the number of titles should be the same.'
    
    display = Display(visible=0, size=resolution)
    display.start()

    model = smplx.create(
        model_folder, 
        model_type='smplx',
        gender='NEUTRAL_2020', 
        use_face_contour=False,
        num_betas=300,
        num_expression_coeffs=100,
        ext='npz', 
        use_pca=False
    ).cuda()
    
    data_np_body_list = [
        np.load(npz_path, allow_pickle=True) 
        for npz_path in npz_path_list
    ]
    faces = np.load(f'{model_folder}/smplx/SMPLX_NEUTRAL_2020.npz', allow_pickle=True)['f']
    
    if not os.path.exists(output_dir):
        os.makedirs(output_dir)

    def get_vertices_all(data_np_body):
        n = data_np_body['poses'].shape[0]
        # beta = th.from_numpy(data_np_body['betas']).float().unsqueeze(0).cuda()
        # beta = beta.repeat(n, 1)
        # expression = th.from_numpy(data_np_body['expressions'][:n]).float().cuda()
        betas = th.zeros(n, 300).float().cuda()
        expression = th.zeros(n, 100).float().cuda()
        jaw_pose = th.from_numpy(data_np_body['poses'][:n, 66:69]).float().cuda()
        pose = th.from_numpy(data_np_body['poses'][:n]).float().cuda()
        transl = th.from_numpy(data_np_body['trans'][:n]).float().cuda()
        output = model(
            betas=betas,
            expression=expression,
            transl=transl,
            jaw_pose=jaw_pose,
            global_orient=pose[:,:3], 
            body_pose=pose[:,3:21*3+3], 
            left_hand_pose=pose[:,25*3:40*3], 
            right_hand_pose=pose[:,40*3:55*3],
            leye_pose=pose[:, 69:72], 
            reye_pose=pose[:, 72:75],
            return_verts=True
        )
        return output['vertices'].cpu().detach().numpy()
    
    vertices_all_list = [
        get_vertices_all(data_np_body) 
        for data_np_body in data_np_body_list
    ]

    if debug:
        seconds = 1
    else:
        seconds = min([vertices_all.shape[0] for vertices_all in vertices_all_list])//30

    generate_images_multi(
        int(seconds*30), 
        vertices_all_list, 
        title_list,
        faces,
        output_dir,
    )
    filenames = [f'{output_dir}frame_{i}.png' for i in range(int(seconds*30))]

    images = [imageio.imread(filename) for filename in filenames]
    imageio.mimsave(f'{output_dir}raw_{output_name}.mp4', images, fps=30)
    for filename in filenames:
        os.remove(filename)
        
    video = mp.VideoFileClip(f'{output_dir}raw_{output_name}.mp4')
    audio = mp.AudioFileClip(audio_path)
    if audio.duration > video.duration:
        audio = audio.subclip(0, video.duration)
    final_clip = video.set_audio(audio)
    final_clip.write_videofile(f'{output_dir}{output_name}.mp4')
    os.remove(f'{output_dir}raw_{output_name}.mp4')
    
    display.stop()

Sequence of SMPLX poses #83

Description

Metadata

Metadata

Assignees

Labels

Type

Projects

Milestone

Relationships

Development

Issue actions