diff --git a/.gitignore b/.gitignore index 5613033..cf8b242 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,8 @@ notebooks/transientFrames/** .vscode -test +test/* +!test/z-scene-properties # Byte-compiled / optimized / DLL files __pycache__/ diff --git a/mitransient/__init__.py b/mitransient/__init__.py index d4b790f..c744b32 100644 --- a/mitransient/__init__.py +++ b/mitransient/__init__.py @@ -1,24 +1,3 @@ -""" -TODO(diego): -- Import mitsuba, if it's not available then raise an exception - telling the user to source the setpath.sh of the mitsuba installation - or to install mitsuba using pip. -- Ensure that mitransient has autocompletion with the same attributes as mitsuba - Needs testing: set the __all__ or __dict__ variables to equal mitsuba's - __all__ or __dict__ variables, but make sure that our own functions - overwrite Mitsuba's (e.g. our sensor class) -- Check other files' __init__.py files to see if their contents can be moved - to the same function -""" - - -try: - import mitsuba -except ModuleNotFoundError: - raise Exception( - 'The mitsuba installation could not be found. ' - 'Please install it using pip or source the setpath.sh file of your Mitsuba installation.') - from .integrators import * from .render import * from .sensors import * diff --git a/mitransient/integrators/common.py b/mitransient/integrators/common.py index 84313f1..88efa20 100644 --- a/mitransient/integrators/common.py +++ b/mitransient/integrators/common.py @@ -1,9 +1,3 @@ -""" -TODO(diego): Figure out how to handle large numbers of samples per pixel -so that the progress bar updates more frequently -(similar to mitsuba2-transient-nlos's multiple passes) -""" - # Delayed parsing of type annotations from __future__ import annotations as __annotations__ @@ -33,7 +27,7 @@ class TransientADIntegrator(ADIntegrator): 1, then path generation many randomly cease after encountering directly visible surfaces. (Default: 5) """ - # TODO: Add documentation for other parameters + # FIXME: Add documentation for other parameters # note that temporal bins, exposure, initial time are measured in optical path length def __init__(self, props=mi.Properties()): @@ -41,9 +35,19 @@ def __init__(self, props=mi.Properties()): # imported: max_depth and rr_depth + # FIXME(diego): set these values in the XML file + # self.transient_block = None + # self.temporal_bins = props.get('temporal_bins', 128) + # self.exposure = props.get('exposure', 3.5) + # self.initial_time = props.get('initial_time', 0) + # self.camera_unwarp = props.get('camera_unwarp', False) + # self.temporal_filter = props.get('temporal_filter', '') + # self.gaussian_stddev = props.get('gaussian_stddev', 2.0) + # self.progressive = props.get('progressive', 0.0) + self.transient_block = None - self.temporal_bins = props.get('temporal_bins', 128) - self.exposure = props.get('exposure', 3.5) + self.temporal_bins = props.get('temporal_bins', 2048) + self.exposure = props.get('exposure', 0.003) self.initial_time = props.get('initial_time', 0) self.camera_unwarp = props.get('camera_unwarp', False) self.temporal_filter = props.get('temporal_filter', '') @@ -51,7 +55,7 @@ def __init__(self, props=mi.Properties()): self.progressive = props.get('progressive', 0.0) def to_string(self): - # TODO add other parameters + # FIXME add other parameters return f'{type(self).__name__}[max_depth = {self.max_depth},' \ f' rr_depth = { self.rr_depth }]' @@ -565,7 +569,7 @@ def sample(self, This mask array can optionally be used to indicate that some of the rays are disabled. - TODO(diego): Parameter ``add_transient_f`` (and document type above) + FIXME(diego): Parameter ``add_transient_f`` (and document type above) or probably refer to non-transient RB The function returns a tuple ``(spec, valid, state_out)`` where @@ -583,8 +587,7 @@ def sample(self, 'It should be implemented by subclasses that ' 'specialize the abstract RBIntegrator interface.') -# Prioritizes RBIntegrator functions over TransientADIntegrator - class TransientRBIntegrator(RBIntegrator, TransientADIntegrator): + # Prioritizes RBIntegrator functions over TransientADIntegrator pass diff --git a/mitransient/integrators/transient_prb_volpath.py b/mitransient/integrators/transient_prb_volpath.py index d773184..8e5f021 100644 --- a/mitransient/integrators/transient_prb_volpath.py +++ b/mitransient/integrators/transient_prb_volpath.py @@ -1,10 +1,11 @@ -from __future__ import annotations # Delayed parsing of type annotations +from __future__ import annotations # Delayed parsing of type annotations import drjit as dr import mitsuba as mi from .common import TransientRBIntegrator, mis_weight + def index_spectrum(spec, idx): m = spec[0] if mi.is_rgb: @@ -12,6 +13,7 @@ def index_spectrum(spec, idx): m[dr.eq(idx, 2)] = spec[2] return m + class TransientPRBVolpathIntegrator(TransientRBIntegrator): """ This class implements a volumetric Path Replay Backpropagation (PRB) integrator @@ -38,6 +40,7 @@ class TransientPRBVolpathIntegrator(TransientRBIntegrator): for details on PRB and differentiable delta tracking. """ + def __init__(self, props=mi.Properties()): super().__init__(props) self.max_depth = props.get('max_depth', -1) @@ -59,7 +62,8 @@ def prepare_scene(self, scene): # Enable NEE if a medium specifically asks for it self.use_nee = self.use_nee or medium.use_emitter_sampling() self.nee_handle_homogeneous = self.nee_handle_homogeneous or medium.is_homogeneous() - self.handle_null_scattering = self.handle_null_scattering or (not medium.is_homogeneous()) + self.handle_null_scattering = self.handle_null_scattering or ( + not medium.is_homogeneous()) self.is_prepared = True # By default enable always NEE in case there are surfaces self.use_nee = True @@ -73,17 +77,18 @@ def sample(self, state_in: Optional[mi.Spectrum], active: mi.Bool, add_transient, - **kwargs # Absorbs unused arguments - ) -> Tuple[mi.Spectrum, - mi.Bool, mi.Spectrum]: + **kwargs # Absorbs unused arguments + ) -> Tuple[mi.Spectrum, + mi.Bool, mi.Spectrum]: self.prepare_scene(scene) is_primal = mode == dr.ADMode.Primal ray = mi.Ray3f(ray) depth = mi.UInt32(0) # Depth of current vertex - L = mi.Spectrum(0 if is_primal else state_in) # Radiance accumulator - δL = mi.Spectrum(δL if δL is not None else 0) # Differential/adjoint radiance + L = mi.Spectrum(0 if is_primal else state_in) # Radiance accumulator + # Differential/adjoint radiance + δL = mi.Spectrum(δL if δL is not None else 0) throughput = mi.Spectrum(1) # Path throughput weight η = mi.Float(1) # Index of refraction active = mi.Bool(active) @@ -112,16 +117,17 @@ def sample(self, valid_ray = mi.Bool(False) specular_chain = mi.Bool(True) - if mi.is_rgb: # Sample a color channel to sample free-flight distances + if mi.is_rgb: # Sample a color channel to sample free-flight distances n_channels = dr.size_v(mi.Spectrum) - channel = dr.minimum(n_channels * sampler.next_1d(active), n_channels - 1) + channel = dr.minimum( + n_channels * sampler.next_1d(active), n_channels - 1) loop = mi.Loop(name=f"Path Replay Backpropagation ({mode.name})", - state=lambda: (sampler, active, depth, ray, medium, si, - throughput, L, needs_intersection, - last_scatter_event, specular_chain, η, - last_scatter_direction_pdf, valid_ray, - distance)) + state=lambda: (sampler, active, depth, ray, medium, si, + throughput, L, needs_intersection, + last_scatter_event, specular_chain, η, + last_scatter_direction_pdf, valid_ray, + distance)) while loop(active): active &= dr.any(dr.neq(throughput, 0.0)) q = dr.minimum(dr.max(throughput) * dr.sqr(η), 0.99) @@ -129,7 +135,8 @@ def sample(self, active &= (sampler.next_1d(active) < q) | ~perform_rr throughput[perform_rr] = throughput * dr.rcp(q) - active_medium = active & dr.neq(medium, None) # TODO this is not necessary + active_medium = active & dr.neq( + medium, None) # TODO this is not necessary active_surface = active & ~active_medium with dr.resume_grad(when=not is_primal): @@ -138,7 +145,8 @@ def sample(self, mei = medium.sample_interaction(ray, u, channel, active_medium) mei.t = dr.detach(mei.t) - ray.maxt[active_medium & medium.is_homogeneous() & mei.is_valid()] = mei.t + ray.maxt[active_medium & medium.is_homogeneous() & + mei.is_valid()] = mei.t intersect = needs_intersection & active_medium si_new = scene.ray_intersect(ray, intersect) si[intersect] = si_new @@ -147,24 +155,31 @@ def sample(self, mei.t[active_medium & (si.t < mei.t)] = dr.inf # Evaluate ratio of transmittance and free-flight PDF - tr, free_flight_pdf = medium.eval_tr_and_pdf(mei, si, active_medium) + tr, free_flight_pdf = medium.eval_tr_and_pdf( + mei, si, active_medium) tr_pdf = index_spectrum(free_flight_pdf, channel) weight = mi.Spectrum(1.0) - weight[active_medium] *= dr.select(tr_pdf > 0.0, tr / dr.detach(tr_pdf), 0.0) + weight[active_medium] *= dr.select(tr_pdf > + 0.0, tr / dr.detach(tr_pdf), 0.0) escaped_medium = active_medium & ~mei.is_valid() active_medium &= mei.is_valid() # Handle null and real scatter events if self.handle_null_scattering: - scatter_prob = index_spectrum(mei.sigma_t, channel) / index_spectrum(mei.combined_extinction, channel) - act_null_scatter = (sampler.next_1d(active_medium) >= scatter_prob) & active_medium + scatter_prob = index_spectrum( + mei.sigma_t, channel) / index_spectrum(mei.combined_extinction, channel) + act_null_scatter = (sampler.next_1d( + active_medium) >= scatter_prob) & active_medium act_medium_scatter = ~act_null_scatter & active_medium - weight[act_null_scatter] *= mei.sigma_n / dr.detach(1 - scatter_prob) + weight[act_null_scatter] *= mei.sigma_n / \ + dr.detach(1 - scatter_prob) # weight[act_null_scatter] *= mei.sigma_n / dr.detach(1 - scatter_prob) weight[act_null_scatter] *= dr.select(dr.neq(dr.detach(1 - scatter_prob), 0.0), - mei.sigma_n / dr.detach(1 - scatter_prob), - 0.0) + mei.sigma_n / + dr.detach( + 1 - scatter_prob), + 0.0) else: scatter_prob = mi.Float(1.0) act_medium_scatter = active_medium @@ -181,7 +196,9 @@ def sample(self, # weight[act_medium_scatter] *= mei.sigma_s / dr.detach(scatter_prob) weight[act_medium_scatter] *= dr.select(dr.neq(dr.detach(scatter_prob), 0.0), - mei.sigma_s / dr.detach(scatter_prob), + mei.sigma_s / + dr.detach( + scatter_prob), 0.0) throughput[active_medium] *= dr.detach(weight) @@ -190,7 +207,8 @@ def sample(self, mei = dr.detach(mei) if not is_primal and dr.grad_enabled(weight): - Lo = dr.detach(dr.select(active_medium | escaped_medium, L / dr.max(1e-8, weight), 0.0)) + Lo = dr.detach( + dr.select(active_medium | escaped_medium, L / dr.max(1e-8, weight), 0.0)) dr.backward(δL * weight * Lo) phase_ctx = mi.PhaseFunctionContext(sampler) @@ -199,13 +217,14 @@ def sample(self, valid_ray |= act_medium_scatter with dr.suspend_grad(): - wo, phase_pdf = phase.sample(phase_ctx, mei, sampler.next_1d(act_medium_scatter), sampler.next_2d(act_medium_scatter), act_medium_scatter) + wo, phase_pdf = phase.sample(phase_ctx, mei, sampler.next_1d( + act_medium_scatter), sampler.next_2d(act_medium_scatter), act_medium_scatter) new_ray = mei.spawn_ray(wo) ray[act_medium_scatter] = new_ray needs_intersection |= act_medium_scatter last_scatter_direction_pdf[act_medium_scatter] = phase_pdf - #--------------------- Surface Interactions --------------------- + # --------------------- Surface Interactions --------------------- active_surface |= escaped_medium intersect = active_surface & needs_intersection si[intersect] = scene.ray_intersect(ray, intersect) @@ -214,12 +233,14 @@ def sample(self, ray_from_camera = active_surface & dr.eq(depth, 0) count_direct = ray_from_camera | specular_chain emitter = si.emitter(scene) - active_e = active_surface & dr.neq(emitter, None) & ~(dr.eq(depth, 0) & self.hide_emitters) + active_e = active_surface & dr.neq(emitter, None) & ~( + dr.eq(depth, 0) & self.hide_emitters) # Get the PDF of sampling this emitter using next event estimation ds = mi.DirectionSample3f(scene, si, last_scatter_event) if self.use_nee: - emitter_pdf = scene.pdf_emitter_direction(last_scatter_event, ds, active_e) + emitter_pdf = scene.pdf_emitter_direction( + last_scatter_event, ds, active_e) else: emitter_pdf = 0.0 emitted = emitter.eval(si, active_e) @@ -229,7 +250,8 @@ def sample(self, if not is_primal and dr.grad_enabled(contrib): dr.backward(δL * contrib) - add_transient(contrib, distance + dr.select(si.is_valid(), ds.dist, 0.0) * η, ray.wavelengths, active_e) + add_transient(contrib, distance + dr.select(si.is_valid(), + ds.dist, 0.0) * η, ray.wavelengths, active_e) active_surface &= si.is_valid() ctx = mi.BSDFContext() @@ -237,7 +259,8 @@ def sample(self, # --------------------- Emitter sampling --------------------- if self.use_nee: - active_e_surface = active_surface & mi.has_flag(bsdf.flags(), mi.BSDFFlags.Smooth) & (depth + 1 < self.max_depth) + active_e_surface = active_surface & mi.has_flag( + bsdf.flags(), mi.BSDFFlags.Smooth) & (depth + 1 < self.max_depth) sample_emitters = mei.medium.use_emitter_sampling() specular_chain &= ~act_medium_scatter specular_chain |= act_medium_scatter & ~sample_emitters @@ -247,34 +270,43 @@ def sample(self, ref_interaction[act_medium_scatter] = mei ref_interaction[active_surface] = si nee_sampler = sampler if is_primal else sampler.clone() - emitted, ds = self.sample_emitter(ref_interaction, scene, sampler, medium, channel, active_e, mode=dr.ADMode.Primal) + emitted, ds = self.sample_emitter( + ref_interaction, scene, sampler, medium, channel, active_e, mode=dr.ADMode.Primal) # Query the BSDF for that emitter-sampled direction - bsdf_val, bsdf_pdf = bsdf.eval_pdf(ctx, si, si.to_local(ds.d), active_e_surface) - phase_val = phase.eval(phase_ctx, mei, ds.d, active_e_medium) - nee_weight = dr.select(active_e_surface, bsdf_val, phase_val) - nee_directional_pdf = dr.select(ds.delta, 0.0, dr.select(active_e_surface, bsdf_pdf, phase_val)) - - contrib = throughput * nee_weight * mis_weight(ds.pdf, nee_directional_pdf) * emitted + bsdf_val, bsdf_pdf = bsdf.eval_pdf( + ctx, si, si.to_local(ds.d), active_e_surface) + phase_val = phase.eval( + phase_ctx, mei, ds.d, active_e_medium) + nee_weight = dr.select( + active_e_surface, bsdf_val, phase_val) + nee_directional_pdf = dr.select(ds.delta, 0.0, dr.select( + active_e_surface, bsdf_pdf, phase_val)) + + contrib = throughput * nee_weight * \ + mis_weight(ds.pdf, nee_directional_pdf) * emitted L[active_e] += dr.detach(contrib if is_primal else -contrib) if not is_primal: self.sample_emitter(ref_interaction, scene, nee_sampler, - medium, channel, active_e, adj_emitted=contrib, δL=δL, mode=mode) + medium, channel, active_e, adj_emitted=contrib, δL=δL, mode=mode) if dr.grad_enabled(nee_weight) or dr.grad_enabled(emitted): dr.backward(δL * contrib) - add_transient(contrib, distance + ds.dist * η, ray.wavelengths, active_e) + add_transient(contrib, distance + ds.dist * + η, ray.wavelengths, active_e) # ----------------------- BSDF sampling ---------------------- with dr.suspend_grad(): bs, bsdf_weight = bsdf.sample(ctx, si, sampler.next_1d(active_surface), - sampler.next_2d(active_surface), active_surface) + sampler.next_2d(active_surface), active_surface) active_surface &= bs.pdf > 0 bsdf_eval = bsdf.eval(ctx, si, bs.wo, active_surface) if not is_primal and dr.grad_enabled(bsdf_eval): - Lo = bsdf_eval * dr.detach(dr.select(active, L / dr.max(1e-8, bsdf_eval), 0.0)) + Lo = bsdf_eval * \ + dr.detach( + dr.select(active, L / dr.max(1e-8, bsdf_eval), 0.0)) if mode == dr.ADMode.Backward: dr.backward_from(δL * Lo) else: @@ -286,19 +318,23 @@ def sample(self, ray[active_surface] = bsdf_ray needs_intersection |= active_surface - non_null_bsdf = active_surface & ~mi.has_flag(bs.sampled_type, mi.BSDFFlags.Null) + non_null_bsdf = active_surface & ~mi.has_flag( + bs.sampled_type, mi.BSDFFlags.Null) depth[non_null_bsdf] += 1 # update the last scatter PDF event if we encountered a non-null scatter event last_scatter_event[non_null_bsdf] = si last_scatter_direction_pdf[non_null_bsdf] = bs.pdf - distance = distance + dr.select(si.is_valid() & ~mei.is_valid(), si.t, 0.0) * η + distance = distance + \ + dr.select(si.is_valid() & ~mei.is_valid(), si.t, 0.0) * η distance[non_null_bsdf] = distance + si.t * η valid_ray |= non_null_bsdf - specular_chain |= non_null_bsdf & mi.has_flag(bs.sampled_type, mi.BSDFFlags.Delta) - specular_chain &= ~(active_surface & mi.has_flag(bs.sampled_type, mi.BSDFFlags.Smooth)) + specular_chain |= non_null_bsdf & mi.has_flag( + bs.sampled_type, mi.BSDFFlags.Delta) + specular_chain &= ~(active_surface & mi.has_flag( + bs.sampled_type, mi.BSDFFlags.Smooth)) has_medium_trans = active_surface & si.is_medium_transition() medium[has_medium_trans] = si.target_medium(ray.d) active &= (active_surface | active_medium) @@ -313,7 +349,8 @@ def sample_emitter(self, ref_interaction, scene, sampler, medium, channel, active = mi.Bool(active) medium = dr.select(active, medium, dr.zeros(mi.MediumPtr)) - ds, emitter_val = scene.sample_emitter_direction(ref_interaction, sampler.next_2d(active), False, active) + ds, emitter_val = scene.sample_emitter_direction( + ref_interaction, sampler.next_2d(active), False, active) ds = dr.detach(ds) invalid = dr.eq(ds.pdf, 0.0) emitter_val[invalid] = 0.0 @@ -328,20 +365,23 @@ def sample_emitter(self, ref_interaction, scene, sampler, medium, channel, state=lambda: (sampler, active, medium, ray, total_dist, needs_intersection, si, transmittance)) while loop(active): - remaining_dist = ds.dist * (1.0 - mi.math.ShadowEpsilon) - total_dist + remaining_dist = ds.dist * \ + (1.0 - mi.math.ShadowEpsilon) - total_dist ray.maxt = dr.detach(remaining_dist) active &= remaining_dist > 0.0 # This ray will not intersect if it reached the end of the segment needs_intersection &= active - si[needs_intersection] = scene.ray_intersect(ray, needs_intersection) + si[needs_intersection] = scene.ray_intersect( + ray, needs_intersection) needs_intersection &= False active_medium = active & dr.neq(medium, None) active_surface = active & ~active_medium # Handle medium interactions / transmittance - mei = medium.sample_interaction(ray, sampler.next_1d(active_medium), channel, active_medium) + mei = medium.sample_interaction(ray, sampler.next_1d( + active_medium), channel, active_medium) mei.t[active_medium & (si.t < mei.t)] = dr.inf mei.t = dr.detach(mei.t) @@ -351,7 +391,8 @@ def sample_emitter(self, ref_interaction, scene, sampler, medium, channel, if self.nee_handle_homogeneous: active_homogeneous = active_medium & medium.is_homogeneous() mei.t[active_homogeneous] = dr.minimum(remaining_dist, si.t) - tr_multiplier[active_homogeneous] = medium.eval_tr_and_pdf(mei, si, active_homogeneous)[0] + tr_multiplier[active_homogeneous] = medium.eval_tr_and_pdf( + mei, si, active_homogeneous)[0] mei.t[active_homogeneous] = dr.inf escaped_medium = active_medium & ~mei.is_valid() @@ -373,11 +414,14 @@ def sample_emitter(self, ref_interaction, scene, sampler, medium, channel, tr_multiplier[active_surface] = tr_multiplier * bsdf_val if not is_primal and dr.grad_enabled(tr_multiplier): - active_adj = (active_surface | active_medium) & (tr_multiplier > 0.0) - dr.backward(tr_multiplier * dr.detach(dr.select(active_adj, δL * adj_emitted / tr_multiplier, 0.0))) + active_adj = (active_surface | active_medium) & ( + tr_multiplier > 0.0) + dr.backward(tr_multiplier * dr.detach(dr.select(active_adj, + δL * adj_emitted / tr_multiplier, 0.0))) # transmittance *= dr.detach(tr_multiplier) - transmittance *= dr.select(dr.isfinite(dr.detach(tr_multiplier)), dr.detach(tr_multiplier), 0.0) + transmittance *= dr.select(dr.isfinite(dr.detach(tr_multiplier)), + dr.detach(tr_multiplier), 0.0) # Update the ray with new origin & t parameter new_ray = si.spawn_ray(mi.Vector3f(ray.d)) @@ -386,7 +430,8 @@ def sample_emitter(self, ref_interaction, scene, sampler, medium, channel, needs_intersection |= active_surface # Continue tracing through scene if non-zero weights exist - active &= (active_medium | active_surface) & dr.any(dr.neq(transmittance, 0.0)) + active &= (active_medium | active_surface) & dr.any( + dr.neq(transmittance, 0.0)) total_dist[active] += dr.select(active_medium, mei.t, si.t) # If a medium transition is taking place: Update the medium pointer @@ -417,4 +462,5 @@ def firstSurface(self, scene, ray_, active_): return distance -mi.register_integrator("transient_prbvolpath", lambda props: TransientPRBVolpathIntegrator(props)) +mi.register_integrator("transient_prbvolpath", + lambda props: TransientPRBVolpathIntegrator(props)) diff --git a/mitransient/integrators/transientnlospath.py b/mitransient/integrators/transientnlospath.py new file mode 100644 index 0000000..e6d36e7 --- /dev/null +++ b/mitransient/integrators/transientnlospath.py @@ -0,0 +1,546 @@ +from __future__ import annotations # Delayed parsing of type annotations + +import drjit as dr +import mitsuba as mi + +from mitransient.integrators.common import TransientRBIntegrator, mis_weight + +from mitsuba import Log, LogLevel +from mitsuba.math import ShadowEpsilon +from typing import Tuple, Optional + + +class TransientNLOSPath(TransientRBIntegrator): + # FIXME(diego): docs + r""" + .. _integrator-prb: + + Path Replay Backpropagation (:monosp:`prb`) + ------------------------------------------- + + .. pluginparameters:: + + * - max_depth + - |int| + - Specifies the longest path depth in the generated output image (where -1 + corresponds to :math:`\infty`). A value of 1 will only render directly + visible light sources. 2 will lead to single-bounce (direct-only) + illumination, and so on. (Default: 6) + + * - rr_depth + - |int| + - Specifies the path depth, at which the implementation will begin to use + the *russian roulette* path termination criterion. For example, if set to + 1, then path generation many randomly cease after encountering directly + visible surfaces. (Default: 5) + + This plugin implements a basic Path Replay Backpropagation (PRB) integrator + with the following properties: + + - Emitter sampling (a.k.a. next event estimation). + + - Russian Roulette stopping criterion. + + - No reparameterization. This means that the integrator cannot be used for + shape optimization (it will return incorrect/biased gradients for + geometric parameters like vertex positions.) + + - Detached sampling. This means that the properties of ideal specular + objects (e.g., the IOR of a glass vase) cannot be optimized. + + See ``prb_basic.py`` for an even more reduced implementation that removes + the first two features. + + See the papers :cite:`Vicini2021` and :cite:`Zeltner2021MonteCarlo` + for details on PRB, attached/detached sampling, and reparameterizations. + + .. tabs:: + + .. code-tab:: python + + 'type': 'prb', + 'max_depth': 8 + """ + + def __init__(self, props: mi.Properties): + super().__init__(props) + + self.filter_depth = props.get('filter_depth', -1) + if self.filter_depth != -1 and self.max_depth != -1: + if self.filter_depth >= self.max_depth: + Log(LogLevel.Warn, + 'You have set filter_depth >= max_depth. ' + 'This will cause the final image to be all zero.') + self.discard_direct_paths = props.get('discard_direct_paths', False) + self.laser_sampling = props.get('nlos_laser_sampling', False) + self.hg_sampling = props.get('nlos_hidden_geometry_sampling', False) + self.hg_sampling_do_rroulette = ( + props.get('nlos_hidden_geometry_sampling_do_rroulette', False) + and + self.hg_sampling + ) + self.hg_sampling_includes_relay_wall = ( + props.get('nlos_hidden_geometry_sampling_includes_relay_wall', False) + and + self.hg_sampling + ) + + def prepare_transient(self, scene: mi.Scene, sensor: mi.Sensor): + super().prepare_transient(scene, sensor) + + import numpy as np + + total_pdf = 0.0 + # same as m_shapes, but excluding relay wall objects (i.e. objects + # that are attached to a sensor) + self.hidden_geometries = [] + # cumulative PDF of m_hidden_geometries + # m_hidden_geometries_pdf[i] = P(area-weighted random index <= i) + # e.g. if two hidden geometry objects with areas A_1 = 1 and A_2 = 2 + # then hidden_geometries_pdf = {0.33f, 1.0f} + self.hidden_geometries_cpdf = [] + for shape in scene.shapes(): + is_relay_wall = shape.sensor() == sensor + if not self.hg_sampling_includes_relay_wall and is_relay_wall: + continue + self.hidden_geometries.append(shape) + self.hidden_geometries_cpdf.append( + total_pdf + shape.surface_area()) + total_pdf += shape.surface_area() + + self.hidden_geometries_cpdf = np.array( + self.hidden_geometries_cpdf) / total_pdf + + def _sample_hidden_geometry_position( + self, ref: mi.Interaction3f, + sample2: mi.Point2f, active: mi.Mask) -> mi.PositionSample3f: + """ + For non-line of sight scenes, sample a point in the hidden + geometry's surface area. The "hidden geometry" is defined + as any object that does not contain an \ref nloscapturemeter + plugin (i.e. every object but the relay wall) + + Parameters + ---------- + ref + A reference point somewhere within the scene + + sample2 + A uniformly distributed 2D vector + + active + A boolean mask + + Returns + ------- + Position sampling record + """ + + if len(self.hidden_geometries) == 0: + return dr.zeros(mi.PositionSample3f) + + if len(self.hidden_geometries) == 1: + return self.hidden_geometries[0].sample_position( + ref.time, sample2, active) + + index = mi.UInt(0) + active_cpdf = mi.Mask(active) + for cpdf in self.hidden_geometries_cpdf: + active_cpdf[sample2.x < cpdf] = False + index[active] += 1 + + cpdf_before = dr.select( + index == 0, 0.0, self.hidden_geometries_cpdf[index - 1]) + shape_pdf = self.hidden_geometries_cpdf[index] - cpdf_before + + # Rescale sample.x() to lie in [0, 1) again + sample2.assign(mi.Point2f( + (sample2.x - cpdf_before) / shape_pdf, sample2.y)) + + shape: mi.Shape = dr.gather( + mi.Shape, self.hidden_geometries, index, active) + ps = shape.sample_position(ref.time, sample2, active) + ps.pdf *= shape_pdf + + active &= dr.neq(ps.pdf, 0.0) + + return ps + + def emitter_nee_sample( + self, mode: dr.ADMode, scene: mi.Scene, sampler: mi.Sampler, + si: mi.SurfaceInteraction3f, bsdf: mi.BSDF, bsdf_ctx: mi.BSDFContext, + β: mi.Spectrum, distance: mi.Float, η: mi.Float, depth: mi.UInt, + active_e: mi.Mask, add_transient) -> mi.Spectrum: + ds, emitter_spec = scene.sample_emitter_direction( + ref=si, sample=sampler.next_2d(active_e), test_visibility=True, active=active_e) + active_e &= dr.neq(ds.pdf, 0.0) + + primal = mode == dr.ADMode.Primal + with dr.resume_grad(when=not primal): + if not primal: + # Given the detached emitter sample, *recompute* its + # contribution with AD to enable light source optimization + ds.d = dr.normalize(ds.p - si.p) + emitter_val = scene.eval_emitter_direction(si, ds, active_e) + emitter_spec = dr.select( + dr.neq(ds.pdf, 0), emitter_val / ds.pdf, 0) + dr.disable_grad(ds.d) + + # Query the BSDF for that emitter-sampled direction + wo = si.to_local(ds.d) + bsdf_spec, bsdf_pdf = bsdf.eval_pdf( + ctx=bsdf_ctx, si=si, wo=wo, active=active_e) + + mis = dr.select(ds.delta, 1.0, mis_weight(ds.pdf, bsdf_pdf)) + + Lr_dir = mi.Spectrum(0) + if depth == self.filter_depth or not (self.discard_direct_paths and depth < 2): + Lr_dir[active_e] = β * mis * bsdf_spec * emitter_spec + + add_transient(Lr_dir, distance + ds.dist * η, + si.wavelengths, active_e) + + return Lr_dir + + def emitter_laser_sample( + self, mode: dr.ADMode, scene: mi.Scene, sampler: mi.Sampler, + si: mi.SurfaceInteraction3f, bsdf: mi.BSDF, bsdf_ctx: mi.BSDFContext, + β: mi.Spectrum, distance: mi.Float, η: mi.Float, depth: mi.UInt, + active_e: mi.Mask, add_transient) -> mi.Spectrum: + """ + NLOS scenes only have one laser emitter - standard + emitter sampling techniques do not apply as most + directions do not emit any radiance, it needs to be very + lucky to bsdf sample the exact point that the laser is + illuminating + + this modifies the emitter sampling so instead of directly + sampling the laser we sample(1) the point that the laser + is illuminating and then(2) the laser + """ + # FIXME probably needs some "with dr.resume_grad(when=not primal):" + primal = mode == dr.ADMode.Primal + + # 1. Obtain direction to NLOS illuminated point + # and test visibility with ray_test + d = self.nlos_laser_target - si.p + distance_laser = dr.norm(d) + d /= distance_laser + ray_bsdf = mi.Ray3f(o=si.p, d=d, + maxt=distance_laser * (1.0 - ShadowEpsilon), + time=si.time, + wavelengths=si.wavelengths) + active_e &= not scene.ray_test(ray_bsdf, active_e) + + # 2. Evaluate BSDF to desired direction + wo = si.to_local(d) + bsdf_spec = bsdf.eval(ctx=bsdf_ctx, si=si, wo=wo, active=active_e) + bsdf_spec = si.to_world_mueller(bsdf_spec, -wo, si.wi) + + ray_bsdf.maxt = dr.inf + si_bsdf: mi.SurfaceInteraction3f = scene.ray_intersect( + ray_bsdf, active_e) + active_e &= si_bsdf.is_valid() + active_e &= dr.any(mi.depolarizer(bsdf_spec) > dr.epsilon) + wl = si_bsdf.to_local(-d) + active_e &= mi.Frame3f.cos_theta(wl) > 0.0 + + # NOTE(diego): as points are not randomly chosen, + # we need to account for d^2 and cos term because of + # the solid angle projection of si.p to + # nlos_laser_target. + # This is like a point light, but the extra cos term + # exists as it is not a point light that emits in all + # directions :^) + # The incident cos term at + # nlos_laser_target will be taken into account by + # emitter_nee_sample's bsdf + bsdf_spec *= dr.sqr(dr.rcp(distance_laser)) * mi.Frame3f.cos_theta(wl) + + bsdf_next = si_bsdf.bsdf(ray=ray_bsdf) + + # 3. Combine laser + emitter sampling + return self.emitter_nee_sample( + mode=mode, scene=scene, sampler=sampler, ray=ray_bsdf, si=si_bsdf, + bsdf=bsdf_next, bsdf_ctx=bsdf_ctx, β=β * bsdf_spec, distance=distance + distance_laser * η, η=η, + depth=depth+1, active=active_e, add_transient=add_transient) + + def hidden_geometry_sample( + self, scene: mi.Scene, bsdf: mi.BSDF, bsdf_ctx: mi.BSDFContext, si: mi.SurfaceInteraction3f, + _: mi.Float, sample2: mi.Point2f, active: mi.Mask) -> Tuple[mi.BSDFSample3f, mi.Spectrum]: + # FIXME add this functionality in transientnlos + ps_hg: mi.PositionSample3f = self._sample_hidden_geometry_position( + si=si, sample2=sample2, active=active) + active &= dr.neq(ps_hg.pdf, 0.0) + + d = ps_hg.p - si.p + dist = dr.norm(d) + d /= dist + ray_hg = mi.Ray3f(o=si.p, d=d, time=si.time, + wavelengths=si.wavelengths) + si_hg: mi.SurfaceInteraction3f = scene.ray_intersect(ray_hg, active) + active &= si_hg.is_valid() + + si_test = si_hg + p_test = si_test.p + active_test = mi.Mask(active) + num_intersections = mi.UInt(0) + while num_intersections == 0 or dr.any(active_test): + num_intersections[active_test] += 1 + ray_test = mi.Ray3f(o=p_test, d=d, time=si_test.time, + wavelengths=si_test.wavelengths) + si_test: mi.SurfaceInteraction3f = scene.ray_intersect( + ray_test, active_test) + active_test &= si_test.is_valid() + if num_intersections > 100: + # Some rays get stuck in an infinite loop, creating a cycle + # of p_test points. Just ignore these cases. + active[active_test] = False + active_test = mi.Mask(False) + p_test = si_test.p + + wo = si.to_local(d) + bsdf_spec = bsdf.eval(ctx=bsdf_ctx, si=si, wo=wo, active=active) + bsdf_spec = si.to_world_mueller(bsdf_spec, -wo, si.wi) + + wg = si_hg.to_local(-d) + travel_dist = dr.norm(si_hg.p - si.p) + bsdf_spec *= dr.sqr(dr.rcp(travel_dist)) * mi.Frame3f.cos_theta(wg) + # discard low travel dist, produces high variance + # the intergator will use bsdf sampling instead + active &= travel_dist > 0.5 + active &= dr.any(mi.depolarizer(bsdf_spec) > dr.epsilon) + + bs: mi.BSDFSample3f = dr.zeros(mi.BSDFSample3f) + bs.wo = wo + bs.pdf = ps_hg.pdf * num_intersections + bs.eta = 1.0 + bs.sampled_type = mi.BSDFType.DeltaReflection + bs.sampled_component = 0 + + return bs, dr.select(active and bs.pdf > dr.epsilon, bsdf_spec, 0.0) + + def sample(self, + mode: dr.ADMode, + scene: mi.Scene, + sampler: mi.Sampler, + ray: mi.Ray3f, + δL: Optional[mi.Spectrum], + state_in: Optional[mi.Spectrum], + active: mi.Bool, + max_distance: mi.Float, + add_transient) -> Tuple[mi.Spectrum, + mi.Bool, + mi.Spectrum]: + """ + See ``TransientADIntegrator.sample()`` for a description of this interface and + the role of the various parameters and return values. + """ + + # Rendering a primal image? (vs performing forward/reverse-mode AD) + primal = mode == dr.ADMode.Primal + + # Standard BSDF evaluation context for path tracing + bsdf_ctx = mi.BSDFContext() + + # --------------------- Configure loop state ---------------------- + + # Copy input arguments to avoid mutating the caller's state + ray = mi.Ray3f(dr.detach(ray)) + depth = mi.UInt32(0) # Depth of current vertex + L = mi.Spectrum(0 if primal else state_in) # Radiance accumulator + # Differential/adjoint radiance + δL = mi.Spectrum(δL if δL is not None else 0) + β = mi.Spectrum(1) # Path throughput weight + η = mi.Float(1) # Index of refraction + active = mi.Bool(active) # Active SIMD lanes + distance = mi.Float(0) # Distance of the path + + # Variables caching information from the previous bounce + prev_si = dr.zeros(mi.SurfaceInteraction3f) + prev_bsdf_pdf = mi.Float(1.0) + prev_bsdf_delta = mi.Bool(True) + + if self.camera_unwarp: + # FIXME(diego): remove camera_unwarp in favour of this + raise AssertionError('Use account_first_and_last_bounces instead') + + # Record the following loop in its entirety + loop = mi.Loop(name="Path Replay Backpropagation (%s)" % mode.name, + state=lambda: (sampler, ray, depth, L, δL, β, η, active, + prev_si, prev_bsdf_pdf, prev_bsdf_delta, + distance)) + + # Specify the max. number of loop iterations (this can help avoid + # costly synchronization when when wavefront-style loops are generated) + loop.set_max_iterations(self.max_depth) + + if self.laser_sampling: + emitter_sample_f = self.emitter_laser_sample + else: + emitter_sample_f = self.emitter_nee_sample + + while loop(active): + # Compute a surface interaction that tracks derivatives arising + # from differentiable shape parameters (position, normals, etc.) + # In primal mode, this is just an ordinary ray tracing operation. + + with dr.resume_grad(when=not primal): + si = scene.ray_intersect(ray, + ray_flags=mi.RayFlags.All, + coherent=dr.eq(depth, 0)) + + # Update distance + distance += dr.select(active, si.t, 0.0) * η + active &= distance < max_distance + + # Get the BSDF, potentially computes texture-space differentials + bsdf = si.bsdf(ray) + + # ---------------------- Direct emission ---------------------- + + # Compute MIS weight for emitter sample from previous bounce + ds = mi.DirectionSample3f(scene, si=si, ref=prev_si) + + mis = mis_weight( + prev_bsdf_pdf, + scene.pdf_emitter_direction(prev_si, ds, ~prev_bsdf_delta) + ) + + with dr.resume_grad(when=not primal): + Le = β * mis * ds.emitter.eval(si) + + # Add transient contribution + add_transient(Le, distance, ray.wavelengths, active) + + # ---------------------- Emitter sampling ---------------------- + + # Should we continue tracing to reach one more vertex? + active_next = (depth + 1 < self.max_depth) & si.is_valid() + + # Is emitter sampling even possible on the current vertex? + active_e = active_next & mi.has_flag( + bsdf.flags(), mi.BSDFFlags.Smooth) + + # Uses NEE or laser sampling depending on self.laser_sampling + Lr_dir = emitter_sample_f( + mode, scene, sampler, + si, bsdf, bsdf_ctx, + β, distance, η, depth, + active_e, add_transient) + + # ------------------ Detached BSDF sampling ------------------- + + if self.hg_sampling and self.hg_sampling_do_rroulette: + # choose HG or BSDF sampling with Russian Roulette + hg_prob = mi.Float(0.5) + do_hg_sample = sampler.next_1d(active) < hg_prob + pdf_bsdf_method = dr.select( + do_hg_sample, + hg_prob, + mi.Float(1.0) - hg_prob) + else: + # only one option + do_hg_sample = mi.Mask(self.hg_sampling) + pdf_bsdf_method = mi.Float(1.0) + + if do_hg_sample: + bsdf_sample, bsdf_weight = self.hidden_geometry_sample( + scene, bsdf, + bsdf_ctx, si, sampler.next_1d(), sampler.next_2d(), active_next) + if not do_hg_sample or dr.all(mi.depolarizer(bsdf_sample) < dr.epsilon): + bsdf_sample, bsdf_weight = bsdf.sample( + bsdf_ctx, si, sampler.next_1d(), sampler.next_2d(), active_next) + + # ---- Update loop variables based on current interaction ----- + + L = (L + Le + Lr_dir) if primal else (L - Le - Lr_dir) + ray = si.spawn_ray(si.to_world(bsdf_sample.wo)) + η *= bsdf_sample.eta + β *= bsdf_weight / pdf_bsdf_method + + # Information about the current vertex needed by the next iteration + + prev_si = dr.detach(si, True) + prev_bsdf_pdf = bsdf_sample.pdf + prev_bsdf_delta = mi.has_flag( + bsdf_sample.sampled_type, mi.BSDFFlags.Delta) + + # -------------------- Stopping criterion --------------------- + + # Don't run another iteration if the throughput has reached zero + β_max = dr.max(β) + active_next &= dr.neq(β_max, 0) + + # Russian roulette stopping probability (must cancel out ior^2 + # to obtain unitless throughput, enforces a minimum probability) + rr_prob = dr.minimum(β_max * η**2, .95) + + # Apply only further along the path since, this introduces variance + rr_active = depth >= self.rr_depth + β[rr_active] *= dr.rcp(rr_prob) + rr_continue = sampler.next_1d() < rr_prob + active_next &= ~rr_active | rr_continue + + # ------------------ Differential phase only ------------------ + + if not primal: + with dr.resume_grad(): + # 'L' stores the indirectly reflected radiance at the + # current vertex but does not track parameter derivatives. + # The following addresses this by canceling the detached + # BSDF value and replacing it with an equivalent term that + # has derivative tracking enabled. (nit picking: the + # direct/indirect terminology isn't 100% accurate here, + # since there may be a direct component that is weighted + # via multiple importance sampling) + + # Recompute 'wo' to propagate derivatives to cosine term + wo = si.to_local(ray.d) + + # Re-evaluate BSDF * cos(theta) differentiably + bsdf_val = bsdf.eval(bsdf_ctx, si, wo, active_next) + + # Detached version of the above term and inverse + bsdf_val_det = bsdf_weight * bsdf_sample.pdf + inv_bsdf_val_det = dr.select(dr.neq(bsdf_val_det, 0), + dr.rcp(bsdf_val_det), 0) + + # Differentiable version of the reflected indirect + # radiance. Minor optional tweak: indicate that the primal + # value of the second term is always 1. + Lr_ind = L * \ + dr.replace_grad(1, inv_bsdf_val_det * bsdf_val) + + # Differentiable Monte Carlo estimate of all contributions + Lo = Le + Lr_dir + Lr_ind + + if dr.flag(dr.JitFlag.VCallRecord) and not dr.grad_enabled(Lo): + raise Exception( + "The contribution computed by the differential " + "rendering phase is not attached to the AD graph! " + "Raising an exception since this is usually " + "indicative of a bug (for example, you may have " + "forgotten to call dr.enable_grad(..) on one of " + "the scene parameters, or you may be trying to " + "optimize a parameter that does not generate " + "derivatives in detached PRB.)") + + # Propagate derivatives from/to 'Lo' based on 'mode' + if mode == dr.ADMode.Backward: + dr.backward_from(δL * Lo) + else: + δL += dr.forward_to(Lo) + + depth[si.is_valid()] += 1 + active = active_next + + return ( + L if primal else δL, # Radiance/differential radiance + dr.neq(depth, 0), # Ray validity flag for alpha blending + L # State for the differential phase + ) + + +mi.register_integrator("transient_nlos_path", + lambda props: TransientNLOSPath(props)) diff --git a/mitransient/integrators/transientpathnlos.py b/mitransient/integrators/transientpathnlos.py deleted file mode 100644 index 547da8e..0000000 --- a/mitransient/integrators/transientpathnlos.py +++ /dev/null @@ -1,287 +0,0 @@ -""" -TODO(diego): modify as per mitsuba2-transient-nlos -""" - -from __future__ import annotations # Delayed parsing of type annotations - -import drjit as dr -import mitsuba as mi - -from mitransient.integrators.common import TransientRBIntegrator, mis_weight - - -class TransientPath(TransientRBIntegrator): - r""" - .. _integrator-prb: - - Path Replay Backpropagation (:monosp:`prb`) - ------------------------------------------- - - .. pluginparameters:: - - * - max_depth - - |int| - - Specifies the longest path depth in the generated output image (where -1 - corresponds to :math:`\infty`). A value of 1 will only render directly - visible light sources. 2 will lead to single-bounce (direct-only) - illumination, and so on. (Default: 6) - - * - rr_depth - - |int| - - Specifies the path depth, at which the implementation will begin to use - the *russian roulette* path termination criterion. For example, if set to - 1, then path generation many randomly cease after encountering directly - visible surfaces. (Default: 5) - - This plugin implements a basic Path Replay Backpropagation (PRB) integrator - with the following properties: - - - Emitter sampling (a.k.a. next event estimation). - - - Russian Roulette stopping criterion. - - - No reparameterization. This means that the integrator cannot be used for - shape optimization (it will return incorrect/biased gradients for - geometric parameters like vertex positions.) - - - Detached sampling. This means that the properties of ideal specular - objects (e.g., the IOR of a glass vase) cannot be optimized. - - See ``prb_basic.py`` for an even more reduced implementation that removes - the first two features. - - See the papers :cite:`Vicini2021` and :cite:`Zeltner2021MonteCarlo` - for details on PRB, attached/detached sampling, and reparameterizations. - - .. tabs:: - - .. code-tab:: python - - 'type': 'prb', - 'max_depth': 8 - """ - - def sample(self, - mode: dr.ADMode, - scene: mi.Scene, - sampler: mi.Sampler, - ray: mi.Ray3f, - δL: Optional[mi.Spectrum], - state_in: Optional[mi.Spectrum], - active: mi.Bool, - add_transient, - **kwargs # Absorbs unused arguments - ) -> Tuple[mi.Spectrum, - mi.Bool, mi.Spectrum]: - """ - See ``TransientADIntegrator.sample()`` for a description of this interface and - the role of the various parameters and return values. - """ - - # Rendering a primal image? (vs performing forward/reverse-mode AD) - primal = mode == dr.ADMode.Primal - - # Standard BSDF evaluation context for path tracing - bsdf_ctx = mi.BSDFContext() - - # --------------------- Configure loop state ---------------------- - - # Copy input arguments to avoid mutating the caller's state - ray = mi.Ray3f(dr.detach(ray)) - depth = mi.UInt32(0) # Depth of current vertex - L = mi.Spectrum(0 if primal else state_in) # Radiance accumulator - # Differential/adjoint radiance - δL = mi.Spectrum(δL if δL is not None else 0) - β = mi.Spectrum(1) # Path throughput weight - η = mi.Float(1) # Index of refraction - active = mi.Bool(active) # Active SIMD lanes - distance = mi.Float(0) # Distance of the path - - # Variables caching information from the previous bounce - prev_si = dr.zeros(mi.SurfaceInteraction3f) - prev_bsdf_pdf = mi.Float(1.0) - prev_bsdf_delta = mi.Bool(True) - - if self.camera_unwarp: - si = scene.ray_intersect(mi.Ray3f(ray), - ray_flags=mi.RayFlags.All, - coherent=mi.Mask(True)) - - distance[si.is_valid()] = -si.t - - # Record the following loop in its entirety - loop = mi.Loop(name="Path Replay Backpropagation (%s)" % mode.name, - state=lambda: (sampler, ray, depth, L, δL, β, η, active, - prev_si, prev_bsdf_pdf, prev_bsdf_delta, - distance)) - - # Specify the max. number of loop iterations (this can help avoid - # costly synchronization when when wavefront-style loops are generated) - loop.set_max_iterations(self.max_depth) - - while loop(active): - # Compute a surface interaction that tracks derivatives arising - # from differentiable shape parameters (position, normals, etc.) - # In primal mode, this is just an ordinary ray tracing operation. - - with dr.resume_grad(when=not primal): - si = scene.ray_intersect(ray, - ray_flags=mi.RayFlags.All, - coherent=dr.eq(depth, 0)) - - # Update distance - distance += dr.select(active, si.t, 0.0) * η - - # Get the BSDF, potentially computes texture-space differentials - bsdf = si.bsdf(ray) - - # ---------------------- Direct emission ---------------------- - - # Compute MIS weight for emitter sample from previous bounce - ds = mi.DirectionSample3f(scene, si=si, ref=prev_si) - - mis = mis_weight( - prev_bsdf_pdf, - scene.pdf_emitter_direction(prev_si, ds, ~prev_bsdf_delta) - ) - - with dr.resume_grad(when=not primal): - Le = β * mis * ds.emitter.eval(si) - - # Add transient contribution - add_transient(Le, distance, ray.wavelengths, active) - - # ---------------------- Emitter sampling ---------------------- - - # Should we continue tracing to reach one more vertex? - active_next = (depth + 1 < self.max_depth) & si.is_valid() - - # Is emitter sampling even possible on the current vertex? - active_em = active_next & mi.has_flag( - bsdf.flags(), mi.BSDFFlags.Smooth) - - # If so, randomly sample an emitter without derivative tracking. - ds, em_weight = scene.sample_emitter_direction( - si, sampler.next_2d(), True, active_em) - active_em &= dr.neq(ds.pdf, 0.0) - - with dr.resume_grad(when=not primal): - if not primal: - # Given the detached emitter sample, *recompute* its - # contribution with AD to enable light source optimization - ds.d = dr.normalize(ds.p - si.p) - em_val = scene.eval_emitter_direction(si, ds, active_em) - em_weight = dr.select( - dr.neq(ds.pdf, 0), em_val / ds.pdf, 0) - dr.disable_grad(ds.d) - - # Evaluate BSDF * cos(theta) differentiably - wo = si.to_local(ds.d) - bsdf_value_em, bsdf_pdf_em = bsdf.eval_pdf( - bsdf_ctx, si, wo, active_em) - mis_em = dr.select( - ds.delta, 1, mis_weight(ds.pdf, bsdf_pdf_em)) - Lr_dir = β * mis_em * bsdf_value_em * em_weight - - # Add contribution direct emitter sampling - add_transient(Lr_dir, distance + ds.dist * - η, ray.wavelengths, active) - - # ------------------ Detached BSDF sampling ------------------- - - bsdf_sample, bsdf_weight = bsdf.sample(bsdf_ctx, si, - sampler.next_1d(), - sampler.next_2d(), - active_next) - - # ---- Update loop variables based on current interaction ----- - - L = (L + Le + Lr_dir) if primal else (L - Le - Lr_dir) - ray = si.spawn_ray(si.to_world(bsdf_sample.wo)) - η *= bsdf_sample.eta - β *= bsdf_weight - - # Information about the current vertex needed by the next iteration - - prev_si = dr.detach(si, True) - prev_bsdf_pdf = bsdf_sample.pdf - prev_bsdf_delta = mi.has_flag( - bsdf_sample.sampled_type, mi.BSDFFlags.Delta) - - # -------------------- Stopping criterion --------------------- - - # Don't run another iteration if the throughput has reached zero - β_max = dr.max(β) - active_next &= dr.neq(β_max, 0) - - # Russian roulette stopping probability (must cancel out ior^2 - # to obtain unitless throughput, enforces a minimum probability) - rr_prob = dr.minimum(β_max * η**2, .95) - - # Apply only further along the path since, this introduces variance - rr_active = depth >= self.rr_depth - β[rr_active] *= dr.rcp(rr_prob) - rr_continue = sampler.next_1d() < rr_prob - active_next &= ~rr_active | rr_continue - - # ------------------ Differential phase only ------------------ - - if not primal: - with dr.resume_grad(): - # 'L' stores the indirectly reflected radiance at the - # current vertex but does not track parameter derivatives. - # The following addresses this by canceling the detached - # BSDF value and replacing it with an equivalent term that - # has derivative tracking enabled. (nit picking: the - # direct/indirect terminology isn't 100% accurate here, - # since there may be a direct component that is weighted - # via multiple importance sampling) - - # Recompute 'wo' to propagate derivatives to cosine term - wo = si.to_local(ray.d) - - # Re-evaluate BSDF * cos(theta) differentiably - bsdf_val = bsdf.eval(bsdf_ctx, si, wo, active_next) - - # Detached version of the above term and inverse - bsdf_val_det = bsdf_weight * bsdf_sample.pdf - inv_bsdf_val_det = dr.select(dr.neq(bsdf_val_det, 0), - dr.rcp(bsdf_val_det), 0) - - # Differentiable version of the reflected indirect - # radiance. Minor optional tweak: indicate that the primal - # value of the second term is always 1. - Lr_ind = L * \ - dr.replace_grad(1, inv_bsdf_val_det * bsdf_val) - - # Differentiable Monte Carlo estimate of all contributions - Lo = Le + Lr_dir + Lr_ind - - if dr.flag(dr.JitFlag.VCallRecord) and not dr.grad_enabled(Lo): - raise Exception( - "The contribution computed by the differential " - "rendering phase is not attached to the AD graph! " - "Raising an exception since this is usually " - "indicative of a bug (for example, you may have " - "forgotten to call dr.enable_grad(..) on one of " - "the scene parameters, or you may be trying to " - "optimize a parameter that does not generate " - "derivatives in detached PRB.)") - - # Propagate derivatives from/to 'Lo' based on 'mode' - if mode == dr.ADMode.Backward: - dr.backward_from(δL * Lo) - else: - δL += dr.forward_to(Lo) - - depth[si.is_valid()] += 1 - active = active_next - - return ( - L if primal else δL, # Radiance/differential radiance - dr.neq(depth, 0), # Ray validity flag for alpha blending - L # State for the differential phase - ) - - -mi.register_integrator("transient_path", lambda props: TransientPath(props)) diff --git a/mitransient/render/__init__.py b/mitransient/render/__init__.py index 45569a0..af5fbb1 100644 --- a/mitransient/render/__init__.py +++ b/mitransient/render/__init__.py @@ -22,7 +22,6 @@ if do_reload and not name.startswith('common'): importlib.reload(globals()[name]) else: - print(f'Importing {name}') importlib.import_module(f'.{name}', package=__name__) del name diff --git a/mitransient/render/scene.py b/mitransient/render/scene.py deleted file mode 100644 index 2a657d5..0000000 --- a/mitransient/render/scene.py +++ /dev/null @@ -1,153 +0,0 @@ -""" -TODO -https://github.com/diegoroyo/mitsuba2-transient-nlos/blob/feat-transient/src/librender/scene.cpp - -How to extend the Mitsuba 3 scene class? or maybe just create mitransient.Scene which inherits from mitsuba.Scene? - -======== TWO FUNCTIONS - -/** - * \brief For non-line of sight scenes, sample a point in the hidden - * geometry's surface area. The "hidden geometry" is defined - * as any object that does not contain an \ref nloscapturemeter - * plugin (i.e. every object but the relay wall) - * - * \param ref - * A reference point somewhere within the scene - * - * \param sample_ - * A uniformly distributed 2D vector - * - * \return - * Position sampling record - */ -PositionSample3f sample_hidden_geometry_position(const Interaction3f &ref, - const Point2f &sample_, - Mask active = true) const { - - MTS_MASK_ARGUMENT(active); - - using ShapePtr = replace_scalar_t; - - Point2f sample(sample_); - PositionSample3f ps; - - if (likely(!m_hidden_geometries.empty())) { - if (likely(m_hidden_geometries.size() == 1)) { - // Fast path if there is only one shape - ps = m_hidden_geometries[0]->sample_position(ref.time, sample, - active); - } else { - - UInt32 index = 0; - for (Float cpdf : m_hidden_geometries_cpdf) { - if (sample.x() < cpdf) - break; - index++; - } - - Float cpdf_before = - index == 0 ? 0.f : m_hidden_geometries_cpdf[index - 1]; - Float shape_pdf = m_hidden_geometries_cpdf[index] - cpdf_before; - - // Rescale sample.x() to lie in [0,1) again - sample.x() = (sample.x() - cpdf_before) / shape_pdf; - - ShapePtr shape = - gather(m_hidden_geometries.data(), index, active); - - // Sample a direction towards the emitter - ps = shape->sample_position(ref.time, sample, active); - - // Account for the discrete probability of sampling this shape - ps.pdf *= shape_pdf; - } - - active &= neq(ps.pdf, 0.f); - } else { - ps = zero(); - } - - return ps; -} - -/** - * \brief Evaluate the probability density of the \ref - * sample_hidden_geometry_position() technique given an - * filled-in \ref PositionSample record. - * - * \param ps - * A position sampling record, which specifies the query location. - * - * \return - * The solid angle density expressed of the sample - */ -Float pdf_hidden_geometry_position(const PositionSample3f &ps, - Mask active = true) const { - MTS_MASK_ARGUMENT(active); - using ShapePtr = replace_scalar_t; - - if (likely(m_hidden_geometries.size() == 1)) { - // Fast path if there is only one shape - return m_hidden_geometries[0]->pdf_position(ps, active); - } else { - Float shape_pdf = 1.f; - for (UInt32 i = 1; i < m_hidden_geometries.size(); ++i) { - if (m_hidden_geometries[i] == ps.object) { - shape_pdf = m_hidden_geometries_cpdf[i] - - m_hidden_geometries_cpdf[i - 1]; - break; - } - } - - return reinterpret_array(ps.object)->pdf_position(ps, - active) * - shape_pdf; - } -} - - -// same as m_shapes, but excluding relay wall objects (i.e. objects -// that are attached to a sensor) -std::vector> m_hidden_geometries; -// cumulative PDF of m_hidden_geometries -// m_hidden_geometries_pdf[i] = P(area-weighted random index <= i) -// e.g. if two hidden geometry objects with areas A_1 = 1 and A_2 = 2 -// then hidden_geometries_pdf = {0.33f, 1.0f} -std::vector m_hidden_geometries_cpdf; - -======= Extend scene's __init__ function - -// NOTE(diego): set scene for sensors, needed by NLOSCaptureSensor -for (Sensor *sensor: m_sensors) - sensor->set_scene(this); - -// NOTE(diego): prepare data for hidden geometry sampling -bool include_relay_wall = true; -TransientSamplingIntegrator *tsi = - dynamic_cast *>( - m_integrator.get()); -if (tsi) { - include_relay_wall = - tsi->hidden_geometry_sampling_includes_relay_wall(); -} -Float total_pdf = 0.f; -for (const auto &shape : m_shapes) { - bool is_relay_wall = false; - for (const auto &sensor : m_sensors) { - if (sensor->shape() == shape) { - is_relay_wall = true; - break; - } - } - if (!include_relay_wall && is_relay_wall) { - continue; - } - m_hidden_geometries.push_back(shape); - m_hidden_geometries_cpdf.push_back(total_pdf + shape->surface_area()); - total_pdf += shape->surface_area(); -} -for (Float &pdf : m_hidden_geometries_cpdf) { - pdf /= total_pdf; -} -""" diff --git a/mitransient/render/sensor.py b/mitransient/render/sensor.py deleted file mode 100644 index d8a9280..0000000 --- a/mitransient/render/sensor.py +++ /dev/null @@ -1,3 +0,0 @@ -""" -TODO extend sensor class, add is_nlos_sensor which defaults to false -""" diff --git a/mitransient/render/transient_block.py b/mitransient/render/transient_block.py index 2b30fc2..d6fce19 100644 --- a/mitransient/render/transient_block.py +++ b/mitransient/render/transient_block.py @@ -1,5 +1,5 @@ """ -TODO(diego): For now the TransientADIntegrator creates its own TransientBlock, +FIXME(diego): For now the TransientADIntegrator creates its own TransientBlock, but the TransientBlock is not defined in the XML. Figure out how to export this film so that it fits inside the NLOSCaptureMeter (or just figure out how to put this in the XML) """ @@ -239,7 +239,7 @@ def develop(self, gamma=False, integer=False, raw=False): return res[crop_size] def __str__(self): - # TODO update + # FIXME update # return f'ImageBlockND[size = {self.m_size}]' return f'''ImageBlockND[ size = {self.m_size} @@ -250,4 +250,4 @@ def __str__(self): weigths = {self.m_weights} data = {self.m_data} ] - ''' +''' diff --git a/mitransient/sensors/nloscapturemeter.py b/mitransient/sensors/nloscapturemeter.py index 3447fc8..49e8a6a 100644 --- a/mitransient/sensors/nloscapturemeter.py +++ b/mitransient/sensors/nloscapturemeter.py @@ -10,7 +10,7 @@ class NLOSCaptureMeter(mi.Sensor): """ - TODO docs + FIXME(diego) add docs """ # NOTE(diego): we assume the rays start in a vacuum @@ -197,14 +197,11 @@ def eval(self, si: mi.SurfaceInteraction3f, active: mi.Bool = True) -> mi.Spectr def bbox(self) -> mi.BoundingBox3f: return self.m_shape.bbox() - def is_nlos_sensor(self) -> bool: - return True - def to_string(self): - # TODO update with the rest of parameters + # FIXME(diego) update with the rest of parameters # m_shape, m_emitter from NLOSCaptureMeter, m_film from Sensor, etc. return f'{type(self).__name__}[laser_target = {self.m_laser_target},' \ f' confocal = { self.m_is_confocal }]' -mi.register_sensor('NLOSCaptureMeter', lambda props: NLOSCaptureMeter(props)) +mi.register_sensor('nlos_capture_meter', lambda props: NLOSCaptureMeter(props)) diff --git a/test/z-scene-properties/Z.obj b/test/z-scene-properties/Z.obj new file mode 100644 index 0000000..9ea2e3c --- /dev/null +++ b/test/z-scene-properties/Z.obj @@ -0,0 +1,208 @@ +# Blender v2.93.5 OBJ File: '' +# www.blender.org +mtllib Z.mtl +o hidden_geometry_Z +v -0.364878 0.296774 0.004000 +v 0.400000 0.400000 0.004000 +v -0.364878 0.400000 0.004000 +v -0.364878 0.296774 0.004000 +v 0.164553 0.296774 0.004000 +v 0.400000 0.400000 0.004000 +v 0.164553 0.296774 0.004000 +v -0.164553 -0.296774 0.004000 +v 0.400000 0.400000 0.004000 +v -0.400000 -0.400000 0.004000 +v -0.164553 -0.296774 0.004000 +v 0.164553 0.296774 0.004000 +v -0.400000 -0.400000 0.004000 +v 0.400000 -0.296774 0.004000 +v -0.164553 -0.296774 0.004000 +v -0.400000 -0.400000 0.004000 +v 0.400000 -0.400000 0.004000 +v 0.400000 -0.296774 0.004000 +v -0.364878 0.296774 -0.004000 +v -0.364878 0.400000 -0.004000 +v 0.400000 0.400000 -0.004000 +v -0.364878 0.296774 -0.004000 +v 0.400000 0.400000 -0.004000 +v 0.164553 0.296774 -0.003999 +v 0.164553 0.296774 -0.003999 +v 0.400000 0.400000 -0.004000 +v -0.164553 -0.296774 -0.004000 +v -0.400000 -0.400000 -0.003999 +v 0.164553 0.296774 -0.003999 +v -0.164553 -0.296774 -0.004000 +v -0.400000 -0.400000 -0.003999 +v -0.164553 -0.296774 -0.004000 +v 0.400000 -0.296774 -0.004000 +v -0.400000 -0.400000 -0.003999 +v 0.400000 -0.296774 -0.004000 +v 0.400000 -0.400000 -0.004000 +v 0.400000 0.400000 0.004000 +v -0.164553 -0.296774 0.004000 +v -0.164553 -0.296774 -0.004000 +v 0.400000 0.400000 -0.004000 +v -0.164553 -0.296774 0.004000 +v 0.400000 -0.296774 0.004000 +v 0.400000 -0.296774 -0.004000 +v -0.164553 -0.296774 -0.004000 +v -0.400000 -0.400000 0.004000 +v 0.164553 0.296774 0.004000 +v 0.164553 0.296774 -0.003999 +v -0.400000 -0.400000 -0.003999 +v -0.364878 0.400000 0.004000 +v 0.400000 0.400000 0.004000 +v 0.400000 0.400000 -0.004000 +v -0.364878 0.400000 -0.004000 +v -0.364878 0.296774 0.004000 +v -0.364878 0.400000 0.004000 +v -0.364878 0.400000 -0.004000 +v -0.364878 0.296774 -0.004000 +v 0.400000 -0.400000 0.004000 +v -0.400000 -0.400000 0.004000 +v -0.400000 -0.400000 -0.003999 +v 0.400000 -0.400000 -0.004000 +v 0.164553 0.296774 0.004000 +v -0.364878 0.296774 0.004000 +v -0.364878 0.296774 -0.004000 +v 0.164553 0.296774 -0.003999 +v 0.400000 -0.296774 0.004000 +v 0.400000 -0.400000 0.004000 +v 0.400000 -0.400000 -0.004000 +v 0.400000 -0.296774 -0.004000 +v -0.357368 0.303546 -0.000619 +v -0.357368 0.393227 -0.000619 +v 0.390323 0.395568 -0.000410 +v -0.357368 0.303546 -0.000619 +v 0.390323 0.395568 -0.000410 +v 0.167537 0.298140 -0.003730 +v 0.167537 0.298140 -0.003730 +v 0.390323 0.395568 -0.000410 +v -0.167536 -0.298140 -0.003731 +v -0.390322 -0.395569 -0.000410 +v 0.167537 0.298140 -0.003730 +v -0.167536 -0.298140 -0.003731 +v -0.390322 -0.395569 -0.000410 +v -0.167536 -0.298140 -0.003731 +v 0.392490 -0.303546 -0.000619 +v -0.390322 -0.395569 -0.000410 +v 0.392490 -0.303546 -0.000619 +v 0.392490 -0.393228 -0.000619 +v -0.357368 0.303546 0.000618 +v 0.390323 0.395568 0.000410 +v -0.357368 0.393227 0.000618 +v -0.357368 0.303546 0.000618 +v 0.167537 0.298141 0.003731 +v 0.390323 0.395568 0.000410 +v 0.167537 0.298141 0.003731 +v -0.167536 -0.298140 0.003730 +v 0.390323 0.395568 0.000410 +v -0.390322 -0.395569 0.000410 +v -0.167536 -0.298140 0.003730 +v 0.167537 0.298141 0.003731 +v -0.390322 -0.395569 0.000410 +v 0.392490 -0.303546 0.000618 +v -0.167536 -0.298140 0.003730 +v -0.390322 -0.395569 0.000410 +v 0.392490 -0.393228 0.000618 +v 0.392490 -0.303546 0.000618 +v 0.390323 0.395568 -0.000410 +v 0.390323 0.395568 0.000410 +v -0.167536 -0.298140 0.003730 +v -0.167536 -0.298140 -0.003731 +v -0.167536 -0.298140 -0.003731 +v -0.167536 -0.298140 0.003730 +v 0.392490 -0.303546 0.000618 +v 0.392490 -0.303546 -0.000619 +v -0.390322 -0.395569 -0.000410 +v -0.390322 -0.395569 0.000410 +v 0.167537 0.298141 0.003731 +v 0.167537 0.298140 -0.003730 +v -0.357368 0.393227 -0.000619 +v -0.357368 0.393227 0.000618 +v 0.390323 0.395568 0.000410 +v 0.390323 0.395568 -0.000410 +v -0.357368 0.303546 -0.000619 +v -0.357368 0.303546 0.000618 +v -0.357368 0.393227 0.000618 +v -0.357368 0.393227 -0.000619 +v 0.392490 -0.393228 -0.000619 +v 0.392490 -0.393228 0.000618 +v -0.390322 -0.395569 0.000410 +v -0.390322 -0.395569 -0.000410 +v 0.167537 0.298140 -0.003730 +v 0.167537 0.298141 0.003731 +v -0.357368 0.303546 0.000618 +v -0.357368 0.303546 -0.000619 +v 0.392490 -0.303546 -0.000619 +v 0.392490 -0.303546 0.000618 +v 0.392490 -0.393228 0.000618 +v 0.392490 -0.393228 -0.000619 +vn 0.0000 0.0000 1.0000 +vn -0.0000 -0.0000 -1.0000 +vn 0.7770 -0.6295 0.0000 +vn 0.0000 1.0000 0.0000 +vn -0.7770 0.6295 0.0000 +vn -1.0000 0.0000 0.0000 +vn 0.0000 -1.0000 0.0000 +vn 1.0000 0.0000 0.0000 +vn 0.0003 -0.0000 -1.0000 +vn -0.0054 0.0465 -0.9989 +vn 0.0198 -0.0111 -0.9997 +vn -0.0198 0.0111 -0.9997 +vn 0.0051 -0.0457 -0.9989 +vn -0.0003 -0.0000 -1.0000 +vn 0.0003 0.0000 1.0000 +vn -0.0054 0.0465 0.9989 +vn 0.0198 -0.0111 0.9997 +vn -0.0198 0.0111 0.9997 +vn 0.0051 -0.0457 0.9989 +vn -0.0003 0.0000 1.0000 +vn 0.7793 -0.6267 0.0000 +vn 0.0097 1.0000 0.0000 +vn -0.7793 0.6267 -0.0001 +vn -0.0031 1.0000 0.0000 +vn 0.0030 -1.0000 0.0000 +vn -0.0103 -0.9999 0.0001 +usemtl full_lambertian_hidden +s 1 +f 1//1 2//1 3//1 +f 4//1 5//1 6//1 +f 7//1 8//1 9//1 +f 10//1 11//1 12//1 +f 13//1 14//1 15//1 +f 16//1 17//1 18//1 +f 19//2 20//2 21//2 +f 22//2 23//2 24//2 +f 25//2 26//2 27//2 +f 28//2 29//2 30//2 +f 31//2 32//2 33//2 +f 34//2 35//2 36//2 +f 37//3 38//3 39//3 40//3 +f 41//4 42//4 43//4 44//4 +f 45//5 46//5 47//5 48//5 +f 49//4 50//4 51//4 52//4 +f 53//6 54//6 55//6 56//6 +f 57//7 58//7 59//7 60//7 +f 61//7 62//7 63//7 64//7 +f 65//8 66//8 67//8 68//8 +f 69//9 70//9 71//9 +f 72//10 73//10 74//10 +f 75//11 76//11 77//11 +f 78//12 79//12 80//12 +f 81//13 82//13 83//13 +f 84//14 85//14 86//14 +f 87//15 88//15 89//15 +f 90//16 91//16 92//16 +f 93//17 94//17 95//17 +f 96//18 97//18 98//18 +f 99//19 100//19 101//19 +f 102//20 103//20 104//20 +f 105//21 106//21 107//21 108//21 +f 109//22 110//22 111//22 112//22 +f 113//23 114//23 115//23 116//23 +f 117//24 118//24 119//24 120//24 +f 121//6 122//6 123//6 124//6 +f 125//25 126//25 127//25 128//25 +f 129//26 130//26 131//26 132//26 +f 133//8 134//8 135//8 136//8 diff --git a/test/z-scene-properties/nlos_scene.xml b/test/z-scene-properties/nlos_scene.xml new file mode 100644 index 0000000..a39765d --- /dev/null +++ b/test/z-scene-properties/nlos_scene.xml @@ -0,0 +1,59 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/test/z-scene-properties/scene_properties.py b/test/z-scene-properties/scene_properties.py new file mode 100644 index 0000000..d59330b --- /dev/null +++ b/test/z-scene-properties/scene_properties.py @@ -0,0 +1,15 @@ +import mitsuba as mi +mi.set_variant('llvm_ad_rgb') # nopep8 + +# Extra imports +import drjit as dr +import numpy as np +import matplotlib.pyplot as plt + +import mitransient as mitr + +scene = mi.load_file( + '/media/pleiades/vault/projects/202110-nlos-render/mitsuba3-transient/mitsuba3-transient/test/z-scene-properties/nlos_scene.xml') + +# integrator = scene.integrator() +# transient_integrator.prepare_transient(kernel_scene, 0) diff --git a/test/z-scene-properties/test.sh b/test/z-scene-properties/test.sh new file mode 100755 index 0000000..4214126 --- /dev/null +++ b/test/z-scene-properties/test.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +source /media/pleiades/vault/projects/202110-nlos-render/mitsuba3-transient/mitsuba3/build/setpath.sh +cd /media/pleiades/vault/projects/202110-nlos-render/mitsuba3-transient/mitsuba3-transient +python3 -m pip install . +cd /media/pleiades/vault/projects/202110-nlos-render/mitsuba3-transient/mitsuba3-transient/test/z-scene-properties +python3 scene_properties.py \ No newline at end of file