Jimeng OmniHuman 1.5

OmniHuman 1.5 (the Jimeng-sourced digital-human model) generates video from a single uploaded image plus audio that corresponds to the image. It accepts images of people or other subjects (pets, anime characters, etc.) in any aspect ratio and combines them with audio to produce high-quality video.

The subject's emotion and motion are strongly tied to the audio, and prompts can adjust the scene, actions, and camera work. OmniHuman 1.5 also handles anime and pet subjects well and lets you designate the speaker/subject, making it useful for storytelling, duets, product interaction, comic drama, and more. Compared to the previous generation, OmniHuman 1.5 improves motion naturalness and structural stability noticeably, with stronger expressiveness in subject motion and overall picture quality.

For detailed model documentation, see the Volcengine docs.

Call Examples

SilvaMux provides the same Jimeng OmniHuman 1.5 API as Volcengine; you can use the Volcengine SDK or integrate through your own API client.

Integration goes through the Volcengine-compatible CV API: POST /api/ark?Action=CVSubmitTask|CVProcess|CVGetResult&Version=2022-08-31; AK/SK authentication and signing are described in the Volcengine-compatible section of Asset Management. The Volcengine links below are for looking up OmniHuman-specific input fields; the access address and credentials still come from SilvaMux.

Using the Volcengine Python SDK as an example — install the SDK with pip install volcengine, then run the sample script:

# coding:utf-8
import json
import threading
from time import sleep

from volcengine.ApiInfo import ApiInfo
from volcengine.Credentials import Credentials
from volcengine.base.Service import Service
from volcengine.ServiceInfo import ServiceInfo
from volcengine.visual.VisualService import VisualService

class SilvaMuxVisualService(VisualService):
    def __new__(cls, *args, **kwargs):
        return object.__new__(cls, *args, **kwargs)

    def __init__(self):
        self.service_info = SilvaMuxVisualService.get_service_info()
        self.api_info = SilvaMuxVisualService.get_api_info()
        super(VisualService, self).__init__(self.service_info, self.api_info)

    def get_service_info():
        # Use www.silvamux.com for the CN site; replace with www.silvamux.io for the global site
        service_info = ServiceInfo("www.silvamux.com",
                                   {}, Credentials('', '', 'cv', 'cn-north-1'), 30, 30, 'https')
        return service_info

    def get_api_info():
        api_info = {
            "CVGetResult": ApiInfo("POST", "/api/ark", {"Action": "CVGetResult", "Version": "2022-08-31"}, {}, {}),
            "CVSubmitTask": ApiInfo("POST", "/api/ark", {"Action": "CVSubmitTask", "Version": "2022-08-31"}, {}, {}),
            "CVProcess": ApiInfo("POST", "/api/ark", {"Action": "CVProcess", "Version": "2022-08-31"}, {}, {}),
        }
        return api_info


def get_result(req_key, task_id):
    i = 0
    while True:
        i += 1
        result_resp = visual_service.cv_get_result({
            "req_key": req_key,
            "task_id": task_id
        })
        result_status = result_resp['data']['status']
        print(f"  Query #{i}, status: {result_status}")
        if result_status == "in_queue" or result_status == "generating":
            sleep(3)
            continue
        if result_status == "done":
            if 'data' in result_resp and 'resp_data' in result_resp['data']:
                return json.loads(result_resp['data']['resp_data'])
            elif 'data' in result_resp and 'video_url' in result_resp['data']:
                return result_resp['data']['video_url']
            else:
                print(f"  Failed to parse: {result_resp}")
                raise Exception("result parse failed")
        raise Exception(f"task {result_status}")

if __name__ == '__main__':
    image_url = "https://portal.volccdn.com/obj/volcfe/cloud-universal-doc/upload_7297f5f099cee6b48f5417e47ac8291b.png"
    audio_url = "https://p9-arcosite.byteimg.com/obj/tos-cn-i-goo7wpa0wc/64c66c987973400491c0b487d832537c"
    mask_urls = []

    visual_service = SilvaMuxVisualService()

    # Use the "Volcengine compatible (AK/SK)" credentials created in the console to call the Volcengine-compatible API
    visual_service.set_ak('AKexampleReplaceWithRealAK')
    visual_service.set_sk('SKexampleReplaceWithRealSK')

    print("Step 1: subject detection — skip if you are sure the image contains a human subject")
    step1_resp = visual_service.cv_submit_task({
        "req_key": "jimeng_realman_avatar_picture_create_role_omni_v15",
        "image_url": image_url
    })
    step1_resp_task_id = step1_resp['data']['task_id']
    print(f"  Task ID: {step1_resp_task_id}")

    step1_result = get_result("jimeng_realman_avatar_picture_create_role_omni_v15", step1_resp_task_id)
    if step1_result['status'] != 1:
        raise Exception("No subject detected; the task failed. Try a different image.")

    print("Step 2: subject segmentation — skip if you don't need to designate the speaking subject in the video")
    step2_resp = visual_service.cv_process({
        "req_key": "jimeng_realman_avatar_object_detection",
        "image_url": image_url
    })
    step2_data = json.loads(step2_resp['data']['resp_data'])
    mask_urls = step2_data['object_detection_result']['mask']['url']
    print(f"  Mask list: {mask_urls}")

    print("Step 3: video generation")
    step3_resp = visual_service.cv_submit_task({
        "req_key": "jimeng_realman_avatar_picture_omni_v15",
        "image_url": image_url,
        "mask_url": mask_urls,
        "audio_url": audio_url,
    })
    step3_resp_task_id = step3_resp['data']['task_id']
    print(f"  Task ID: {step3_resp_task_id}")

    step3_result = get_result("jimeng_realman_avatar_picture_omni_v15", step3_resp_task_id)
    print(f"  Result: {step3_result}")

Detailed API documentation: