Skip to content
This repository was archived by the owner on Sep 20, 2024. It is now read-only.

Farm publishing of existing frames #5451

Closed
jakubjezek001 wants to merge 49 commits into from
Closed

Farm publishing of existing frames #5451

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
49 commits
Select commit Hold shift + click to select a range
4454755
traypublisher: publishing via farm wip
jakubjezek001 Aug 9, 2023
cc9d961
colorspace: abstracting publishing functions out of mixin
jakubjezek001 Aug 10, 2023
eb80100
initialisation of expected files module
jakubjezek001 Aug 10, 2023
4d18781
initialisation of frames module
jakubjezek001 Aug 10, 2023
68c115b
updating pipeline publish plugins module
jakubjezek001 Aug 10, 2023
2de6e74
nuke: rewriting collect write to use abstracted utility functions
jakubjezek001 Aug 10, 2023
a8598ad
renaming first / last frame to frame start/end
jakubjezek001 Aug 10, 2023
85fdb28
Merge branch 'develop' into enhancement/OP-6317_abstraction_farm_publ…
jakubjezek001 Aug 10, 2023
812c8e0
fix how version definition order
iLLiCiTiT Aug 10, 2023
a39a10f
added 'folder' to anatomy data
iLLiCiTiT Aug 10, 2023
3e4a7d9
Merge branch 'bugfix/versions-post-fixes' into enhancement/OP-6317_ab…
jakubjezek001 Aug 10, 2023
bf93c90
houdini: implementing expected_files
jakubjezek001 Aug 11, 2023
242c32d
houdini: adding expected files to renderers
jakubjezek001 Aug 11, 2023
b7471ab
houdini: adding expected files to mantra
jakubjezek001 Aug 11, 2023
aeb7852
nuke: updating expected files for farm representation
jakubjezek001 Aug 11, 2023
ea97f14
rename function to more appropriate name based on content
jakubjezek001 Aug 11, 2023
7d5811e
nuke, general: simplifying code for farm publishing representation
jakubjezek001 Aug 11, 2023
ad11460
removing strange imports
jakubjezek001 Aug 11, 2023
118b7de
Merge branch 'develop' into enhancement/OP-6317_abstraction_farm_publ…
jakubjezek001 Aug 11, 2023
c4368af
nuke, general: simplifying collect write
jakubjezek001 Aug 11, 2023
d21824b
traypublisher, general: updating simple instances collector
jakubjezek001 Aug 11, 2023
cc6f9e2
adding exception for farm publishing
jakubjezek001 Aug 12, 2023
819c838
removing colorspace and making frame start and end optional
jakubjezek001 Aug 12, 2023
f90add8
deadline: exception for missing expectedFiles
jakubjezek001 Aug 12, 2023
b1cd2be
farm publishing target plugin and other work
jakubjezek001 Aug 12, 2023
97ef1ed
global farm target collector
jakubjezek001 Aug 14, 2023
e4b5c37
farm targeting update
jakubjezek001 Aug 14, 2023
40a7297
updating submit publish job
jakubjezek001 Aug 14, 2023
83e3d10
removing hardcoded render and prerender family - skeleton instance data
jakubjezek001 Aug 14, 2023
c41e8b7
adding representation level persistent staging dir
jakubjezek001 Aug 14, 2023
9ee389a
adding persistent staging dir to representation
jakubjezek001 Aug 14, 2023
0feea51
adding question for persistency at representation
jakubjezek001 Aug 14, 2023
2d91f42
adding more families configs
jakubjezek001 Aug 14, 2023
769494f
set only enabled farm modules to render target
jakubjezek001 Aug 14, 2023
51a596c
colorspace is not needed
jakubjezek001 Aug 14, 2023
492e9cf
existing frames validation fix of condition
jakubjezek001 Aug 14, 2023
254c260
missing instances
jakubjezek001 Aug 14, 2023
6fbf255
validation of any existing frames without explicit farm publishing
jakubjezek001 Aug 14, 2023
0d863eb
Merge branch 'develop' into enhancement/OP-6317_abstraction_farm_publ…
jakubjezek001 Aug 14, 2023
a3261b3
nuke: slate recalculation fix
jakubjezek001 Aug 14, 2023
e5c8b64
fixing optional arguments condition
jakubjezek001 Aug 14, 2023
68254b5
colorspace: making the plugin more useful at collecting time
jakubjezek001 Aug 14, 2023
874e9fb
Merge branch 'develop' into enhancement/OP-6317_abstraction_farm_publ…
jakubjezek001 Aug 16, 2023
070d3d5
Merge branch 'develop' into enhancement/OP-6317_abstraction_farm_publ…
jakubjezek001 Aug 24, 2023
67a382d
Update openpype/pipeline/colorspace.py
jakubjezek001 Aug 24, 2023
eb850d1
Update openpype/pipeline/colorspace.py
jakubjezek001 Aug 24, 2023
3a83de7
Update openpype/modules/deadline/plugins/publish/submit_publish_job.py
jakubjezek001 Aug 25, 2023
4db8667
Update openpype/modules/deadline/plugins/publish/validate_expected_an…
jakubjezek001 Aug 25, 2023
7388171
Update openpype/pipeline/farm/pyblish_functions.py
jakubjezek001 Aug 25, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion openpype/hosts/aftereffects/plugins/publish/collect_render.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -138,7 +138,6 @@ def get_instances(self, context):
fam = "render.farm"
if fam not in instance.families:
instance.families.append(fam)
instance.toBeRenderedOn = "deadline"
instance.renderer = "aerender"
instance.farm = True # to skip integrate
if "review" in instance.families:
Expand Down
1 change: 0 additions & 1 deletion openpype/hosts/fusion/plugins/publish/collect_render.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,6 @@ def get_instances(self, context):
fam = "render.farm"
if fam not in instance.families:
instance.families.append(fam)
instance.toBeRenderedOn = "deadline"
instance.farm = True # to skip integrate
if "review" in instance.families:
# to skip ExtractReview locally
Expand Down
1 change: 0 additions & 1 deletion openpype/hosts/harmony/plugins/publish/collect_farm_render.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -174,7 +174,6 @@ def get_instances(self, context):
outputFormat=info[1],
outputStartFrame=info[3],
leadingZeros=info[2],
toBeRenderedOn='deadline',
ignoreFrameHandleCheck=True

)
Expand Down
34 changes: 7 additions & 27 deletions openpype/hosts/houdini/plugins/publish/collect_arnold_rop.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
import hou
import pyblish.api

from openpype.pipeline import expected_files
from openpype.hosts.houdini.api import colorspace
from openpype.hosts.houdini.api.lib import (
evalParmNoFrame, get_color_management_preferences)
Expand All @@ -27,6 +28,8 @@ class CollectArnoldROPRenderProducts(pyblish.api.InstancePlugin):
def process(self, instance):

rop = hou.node(instance.data.get("instance_node"))
frame_start = instance.data["frameStart"]
frame_end = instance.data["frameEnd"]

# Collect chunkSize
chunk_size_parm = rop.parm("chunkSize")
Expand All @@ -44,7 +47,8 @@ def process(self, instance):
render_products.append(beauty_product)

files_by_aov = {
"": self.generate_expected_files(instance, beauty_product)
"": expected_files.generate_expected_filepaths(
frame_start, frame_end, beauty_product)
}

num_aovs = rop.evalParm("ar_aovs")
Expand All @@ -61,8 +65,8 @@ def process(self, instance):
aov_product = self.get_render_product_name(default_prefix,
suffix=label)
render_products.append(aov_product)
files_by_aov[label] = self.generate_expected_files(instance,
aov_product)
files_by_aov[label] = expected_files.generate_expected_filepaths(
frame_start, frame_end, aov_product)

for product in render_products:
self.log.debug("Found render product: {}".format(product))
Expand Down Expand Up @@ -109,27 +113,3 @@ def get_render_product_name(self, prefix, suffix):
product_name = prefix

return product_name

def generate_expected_files(self, instance, path):
"""Create expected files in instance data"""

dir = os.path.dirname(path)
file = os.path.basename(path)

if "#" in file:
def replace(match):
return "%0{}d".format(len(match.group()))

file = re.sub("#+", replace, file)

if "%" not in file:
return path

expected_files = []
start = instance.data["frameStart"]
end = instance.data["frameEnd"]
for i in range(int(start), (int(end) + 1)):
expected_files.append(
os.path.join(dir, (file % i)).replace("\\", "/"))

return expected_files
31 changes: 5 additions & 26 deletions openpype/hosts/houdini/plugins/publish/collect_karma_rop.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
import hou
import pyblish.api

from openpype.pipeline import expected_files
from openpype.hosts.houdini.api.lib import (
evalParmNoFrame,
get_color_management_preferences
Expand Down Expand Up @@ -31,6 +32,8 @@ class CollectKarmaROPRenderProducts(pyblish.api.InstancePlugin):
def process(self, instance):

rop = hou.node(instance.data.get("instance_node"))
frame_start = instance.data["frameStart"]
frame_end = instance.data["frameEnd"]

# Collect chunkSize
chunk_size_parm = rop.parm("chunkSize")
Expand All @@ -49,8 +52,8 @@ def process(self, instance):
render_products.append(beauty_product)
Copy link
Copy Markdown
Contributor

@fabiaserra fabiaserra Aug 15, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

similar to @BigRoy 's comment, only the part of the code that collects the chunk size should be under that scope:

# Collect chunkSize
chunk_size_parm = rop.parm("chunkSize")
if chunk_size_parm:
     chunk_size = int(chunk_size_parm.eval())
     instance.data["chunkSize"] = chunk_size
     self.log.debug("Chunk Size: %s" % chunk_size)

default_prefix = evalParmNoFrame(rop, "picture")
render_products = []

Or if we actually require that parm to exist, error out and return early


files_by_aov = {
"beauty": self.generate_expected_files(instance,
beauty_product)
"beauty": expected_files.generate_expected_filepaths(
frame_start, frame_end, beauty_product)
}

filenames = list(render_products)
Expand Down Expand Up @@ -78,27 +81,3 @@ def get_render_product_name(self, prefix, suffix):
product_name = "{}.{}{}".format(prefix_base, suffix, ext)

return product_name

def generate_expected_files(self, instance, path):
"""Create expected files in instance data"""

dir = os.path.dirname(path)
file = os.path.basename(path)

if "#" in file:
def replace(match):
return "%0{}d".format(len(match.group()))

file = re.sub("#+", replace, file)

if "%" not in file:
return path

expected_files = []
start = instance.data["frameStart"]
end = instance.data["frameEnd"]
for i in range(int(start), (int(end) + 1)):
expected_files.append(
os.path.join(dir, (file % i)).replace("\\", "/"))

return expected_files
116 changes: 54 additions & 62 deletions openpype/hosts/houdini/plugins/publish/collect_mantra_rop.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
import re
import os

import hou
import pyblish.api

from openpype.pipeline import expected_files
from openpype.hosts.houdini.api.lib import (
evalParmNoFrame,
get_color_management_preferences
Expand Down Expand Up @@ -31,49 +31,65 @@ class CollectMantraROPRenderProducts(pyblish.api.InstancePlugin):
def process(self, instance):

rop = hou.node(instance.data.get("instance_node"))
frame_start = instance.data["frameStart"]
frame_end = instance.data["frameEnd"]

# Collect chunkSize
chunk_size_parm = rop.parm("chunkSize")
if chunk_size_parm:
chunk_size = int(chunk_size_parm.eval())
instance.data["chunkSize"] = chunk_size
self.log.debug("Chunk Size: %s" % chunk_size)

default_prefix = evalParmNoFrame(rop, "vm_picture")
render_products = []

# Default beauty AOV
beauty_product = self.get_render_product_name(
prefix=default_prefix, suffix=None
if not chunk_size_parm:
return
Comment thread
jakubjezek001 marked this conversation as resolved.

chunk_size = int(chunk_size_parm.eval())
instance.data["chunkSize"] = chunk_size
self.log.debug("Chunk Size: {}".format(chunk_size))

default_prefix = evalParmNoFrame(rop, "vm_picture")
render_products = []

# Default beauty AOV
beauty_product = self.get_render_product_name(
prefix=default_prefix, suffix=None
)
render_products.append(beauty_product)

files_by_aov = {
"beauty": expected_files.generate_expected_filepaths(
frame_start, frame_end, beauty_product)
}

# get the number of AOVs
aov_numbers = rop.evalParm("vm_numaux")
if aov_numbers < 0:
Comment thread
jakubjezek001 marked this conversation as resolved.
return

# get the filenames of the AOVs
for index in range(1, aov_numbers + 1):
var_ = rop.evalParm("vm_variable_plane{:>1}".format(index))

# skip empty variables
if not var_:
continue

aov_name = "vm_filename_plane{:>1}".format(index)
aov_boolean = "vm_usefile_plane{:>1}".format(index)
aov_enabled = rop.evalParm(aov_boolean)
has_aov_path = rop.evalParm(aov_name)

# skip disabled AOVs
if not (has_aov_path and aov_enabled == 1):
continue
Comment thread
jakubjezek001 marked this conversation as resolved.

aov_prefix = evalParmNoFrame(rop, aov_name)
aov_product = self.get_render_product_name(
prefix=aov_prefix, suffix=None
)
render_products.append(beauty_product)

files_by_aov = {
"beauty": self.generate_expected_files(instance,
beauty_product)
}

aov_numbers = rop.evalParm("vm_numaux")
if aov_numbers > 0:
# get the filenames of the AOVs
for i in range(1, aov_numbers + 1):
var = rop.evalParm("vm_variable_plane%d" % i)
if var:
aov_name = "vm_filename_plane%d" % i
aov_boolean = "vm_usefile_plane%d" % i
aov_enabled = rop.evalParm(aov_boolean)
has_aov_path = rop.evalParm(aov_name)
if has_aov_path and aov_enabled == 1:
aov_prefix = evalParmNoFrame(rop, aov_name)
aov_product = self.get_render_product_name(
prefix=aov_prefix, suffix=None
)
render_products.append(aov_product)

files_by_aov[var] = self.generate_expected_files(instance, aov_product) # noqa
render_products.append(aov_product)

files_by_aov[var_] = expected_files.generate_expected_filepaths(
frame_start, frame_end, aov_product)

for product in render_products:
self.log.debug("Found render product: %s" % product)
self.log.debug("Found render product: {}".format(product))

filenames = list(render_products)
instance.data["files"] = filenames
Expand Down Expand Up @@ -101,27 +117,3 @@ def get_render_product_name(self, prefix, suffix):
product_name = prefix_base + "." + suffix + ext

return product_name

def generate_expected_files(self, instance, path):
"""Create expected files in instance data"""

dir = os.path.dirname(path)
file = os.path.basename(path)

if "#" in file:
def replace(match):
return "%0{}d".format(len(match.group()))

file = re.sub("#+", replace, file)

if "%" not in file:
return path

expected_files = []
start = instance.data["frameStart"]
end = instance.data["frameEnd"]
for i in range(int(start), (int(end) + 1)):
expected_files.append(
os.path.join(dir, (file % i)).replace("\\", "/"))

return expected_files
36 changes: 8 additions & 28 deletions openpype/hosts/houdini/plugins/publish/collect_redshift_rop.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
import hou
import pyblish.api

from openpype.pipeline import expected_files
from openpype.hosts.houdini.api.lib import (
evalParmNoFrame,
get_color_management_preferences
Expand Down Expand Up @@ -31,6 +32,8 @@ class CollectRedshiftROPRenderProducts(pyblish.api.InstancePlugin):
def process(self, instance):

rop = hou.node(instance.data.get("instance_node"))
frame_start = instance.data["frameStart"]
frame_end = instance.data["frameEnd"]

# Collect chunkSize
chunk_size_parm = rop.parm("chunkSize")
Expand All @@ -49,8 +52,9 @@ def process(self, instance):
)
render_products.append(beauty_product)
files_by_aov = {
"_": self.generate_expected_files(instance,
beauty_product)}
"_": expected_files.generate_expected_filepaths(
frame_start, frame_end, beauty_product)
}

num_aovs = rop.evalParm("RS_aov")
for index in range(num_aovs):
Expand All @@ -68,8 +72,8 @@ def process(self, instance):
aov_product = self.get_render_product_name(aov_prefix, aov_suffix)
render_products.append(aov_product)

files_by_aov[aov_suffix] = self.generate_expected_files(instance,
aov_product) # noqa
files_by_aov[aov_suffix] = expected_files.generate_expected_filepaths(
Copy link
Copy Markdown

@hound hound bot Aug 25, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

line too long (82 > 79 characters)

frame_start, frame_end, aov_product)

for product in render_products:
self.log.debug("Found render product: %s" % product)
Expand Down Expand Up @@ -115,27 +119,3 @@ def get_render_product_name(self, prefix, suffix):
product_name = prefix

return product_name

def generate_expected_files(self, instance, path):
"""Create expected files in instance data"""

dir = os.path.dirname(path)
file = os.path.basename(path)

if "#" in file:
def replace(match):
return "%0{}d".format(len(match.group()))

file = re.sub("#+", replace, file)

if "%" not in file:
return path

expected_files = []
start = instance.data["frameStart"]
end = instance.data["frameEnd"]
for i in range(int(start), (int(end) + 1)):
expected_files.append(
os.path.join(dir, (file % i)).replace("\\", "/"))

return expected_files
Loading