proc blockdecompose { framecount } {
  set noderank  [parallel noderank]
  set nodecount [parallel nodecount]

  set start [expr round($noderank     * $framecount / $nodecount)]
  set end   [expr round(($noderank+1) * $framecount / $nodecount) - 1]

  return [list $start $end]
}


proc render_one_frame { i userdata } {
  # puts "userdata: '$userdata'"

  set formatstr  [lindex $userdata 0]
  set renderlist [lindex $userdata 1]
  set renderers  [lindex $userdata 2]
  set dirs       [lindex $userdata 3]

  set frameno [lindex $renderlist $i]
  set ::MovieMaker::userframe $frameno
  display update  

  foreach renderer $renderers dir $dirs {
#    puts "Rendering with '$renderer' with dir '$dir'"
    set fname [format $formatstr $frameno]
    set fullpath [file join $dir $fname]
    puts "render $renderer $fullpath"
    if { $renderer != "none" } {
      render $renderer $fullpath
    }
  }
}


proc render_vcr_movie { dirs formatstr renderlist renderers } {
  set noderank  [parallel noderank]
  set nodecount [parallel nodecount]

  set framecount [llength $renderlist]
  set decomposition "dynamic"
#  set decomposition "block"
  
  set userdata {}
  lappend userdata $formatstr
  lappend userdata $renderlist
  lappend userdata $renderers
  lappend userdata $dirs


  # synchronize all nodes at the start
  set t0 [clock milliseconds]

  if { $decomposition == "dynamic" } {
    parallel for 0 [expr $framecount - 1] render_one_frame $userdata
  } else {
    if { $decomposition == "block" } {
      set block [blockdecompose $framecount]
      set start [lindex $block 0]
      set end   [lindex $block 1]
      set skip  1
      set len   [expr $end - $start + 1]
    } elseif { $decomposition == "roundrobin" } {
      set start $noderank
      set end   $framecount
      set skip  $nodecount
      set len   $framecount
    }

    parallel barrier
    puts "Node $noderank render loop: $start to $end ($len frames)..."

    for {set i $start} {$i <  $end} {incr i $skip} {
      render_one_frame $i $userdata
    }
  }

  # This node is done rendering, record runtime and wait for peers
  set t1 [clock milliseconds]
  set t1minust0 [expr $t1 - $t0]
  puts "Node:\[[parallel noderank]\] done: [expr $t1minust0 / 1000.0] sec         "

  # All nodes done, record runtime and print status
  parallel barrier
  set t2 [clock milliseconds]
  set t2minust0 [expr $t2 - $t0]

  # compute the min and max runtimes among all nodes
  # get "min" and "max" from ::tcl::mathfunc namespace
  namespace path {::tcl::mathop ::tcl::mathfunc}
  set part1min [parallel allreduce min $t1minust0]
  set part1max [parallel allreduce max $t1minust0]

  if { [parallel noderank] == 0 } {
    puts "min rendering runtime: [expr $part1min / 1000.0] sec"
    puts "max rendering runtime: [expr $part1max / 1000.0] sec"
  }
}



##
## Run parallel rendering job
##
proc render_movie_parallel { statefiles renderers moviedirs duration continuecheckpoint } {
  ## synchronize all nodes at the start
  parallel barrier
  set globt0 [expr [clock milliseconds] / 1000.0]
  set nodet0 $globt0

  ## disable display updates during state file loading
  display update off
  set env(VMDTSTIMER) 1
  set env(VMDTACHYONWORKGROUP) [parallel noderank]

  ## load code and VCR plugin 
  package require vmdmovie
  package require viewchangerender
#  source vcr_johan.tcl
  set nodet1 [expr [clock milliseconds] / 1000.0]

  ## load VMD state file
  foreach sf $statefiles {
    play $sf
  }
  set nodet2 [expr [clock milliseconds] / 1000.0]

  ## set movie maker parameters etc
  set ::MovieMaker::framerate 30
  set ::MovieMaker::trjstep 1
  set ::MovieMaker::userframe 0
  ::VCR::enableMovieMaker 1
  set ::VCR::movieDuration $duration
  ::VCR::updateMovieMakerDuration
  set ::MovieMaker::numframes [expr int($::VCR::movieDuration*$::MovieMaker::framerate)]

  ## if this is a continuation of a previous rendering run,
  ## check to see what frames are already completed and only render
  ## the ones that do not already exist
  ## NOTE: only node 0 does the filesystem queries to prevent performance
  ##       problems when running with large node counts.  Once node 0 completes
  ##       the checkpoint frames query, it broadcasts the list of frames back
  ##       to all of the other workers
  set renderframelist {}
  if { [parallel noderank] == 0 } {
    foreach moviedir $moviedirs {
      if { ! [file exists $moviedir] } {
        file mkdir $moviedir
      }
    }

    if { $continuecheckpoint } {
      puts "Continuing movie rendering from checkpointed output files."
      puts "Determining list of frames still to be rendered..."
      for { set i 0 } { $i < $::MovieMaker::numframes } { incr i } {
        set needframe 0
        foreach moviedir $moviedirs {
          set checkname [format "%s/frame.%05d.tga" $moviedir $i]
          if { ! [file exists $checkname] || [file size $checkname] == 0 } {
            set needframe 1
          }
        }
        if { $needframe } {
          lappend renderframelist $i
        }
      }
    } else {
      puts "Deleting any existing output files before movie rendering..."
      foreach moviedir $moviedirs {
        file delete [format "%s/frame.*.tga" $moviedir]
#        for { set i 0 } { $i < $::MovieMaker::numframes } { incr i } {
#          set checkname [format "%s/frame.%05d.tga" $moviedir $i]
#          if { [file exists $checkname] } {
#            file delete $checkname
#          } 
#        } 
      }

      for { set i 0 } { $i < $::MovieMaker::numframes } { incr i } {
        lappend renderframelist $i
      }
    }
  }
  set nodet3 [expr [clock milliseconds] / 1000.0]
#  parallel barrier
  set renderframelist [lindex [parallel allgather $renderframelist] 0]
  set nodet4 [expr [clock milliseconds] / 1000.0]

  ## begin rendering frames
  display update on
  render_vcr_movie $moviedirs "frame.%05d.tga" $renderframelist $renderers

  # synchronize all nodes at the end
  parallel barrier
  set globt1 [expr [clock milliseconds] / 1000.0]
  set globt1minust0 [expr $globt1 - $globt0]
  set nodet1minust0 [expr $nodet1 - $nodet0]
  set nodet2minust1 [expr $nodet2 - $nodet1]
  set nodet3minust2 [expr $nodet3 - $nodet2]
  set nodet4minust3 [expr $nodet4 - $nodet3]

  # compute the min and max runtimes among all nodes
  # get "min" and "max" from ::tcl::mathfunc namespace
  namespace path {::tcl::mathop ::tcl::mathfunc}
  set part1min [parallel allreduce min $nodet1minust0]
  set part1max [parallel allreduce max $nodet1minust0]
  set part2min [parallel allreduce min $nodet2minust1]
  set part2max [parallel allreduce max $nodet2minust1]
  set part3min [parallel allreduce min $nodet3minust2]
  set part3max [parallel allreduce max $nodet3minust2]
  set part4min [parallel allreduce min $nodet4minust3]
  set part4max [parallel allreduce max $nodet4minust3]


  if { [parallel noderank] == 0 } {
    puts "min code loading runtime: $part1min sec"
    puts "max code loading runtime: $part1max sec"
    puts "min state loader runtime: $part2min sec"
    puts "max state loader runtime: $part2max sec"
    puts "min frames check runtime: $part3min sec"
    puts "max frames check runtime: $part3max sec"
    puts "min frame allgather time: $part4min sec"
    puts "max frame allgather time: $part4max sec"
    puts "Total runtime for rendering job: $globt1minust0 sec"
  }
}


##
## Setup parallel rendering job
##
proc render_movie { scriptfiles useopenglpbuffer useoptixinternal usetachyoninternal continuecheckpoint filebase } {
  if { [llength scriptfiles] < 1 } {
    return 
  }
 
  set VMDRenderer {}

  if { $useopenglpbuffer } {
    lappend VMDRenderer snapshot
    lappend moviedir    [file join $filebase opengl]
  } 

  if { $useoptixinternal } {
    lappend VMDRenderer TachyonLOptiXInternal
    lappend moviedir    [file join $filebase optix]
  }

  if { $usetachyoninternal } {
    lappend VMDRenderer TachyonInternal
    lappend moviedir    [file join $filebase tachyon]
  }

  # don't render at all, just do the rep work only...
  if { ! (($useoptixinternal) || ($usetachyoninternal) || ($useopenglpbuffer)) } {
    lappend VMDRenderer none
    lappend moviedir    [file join $filebase none]
  }

  set movieduration 105.0
  render_movie_parallel $scriptfiles $VMDRenderer $moviedir $movieduration $continuecheckpoint
}

#puts "Setting QuickSurf min GPU memory flag.."
#set env(VMDQUICKSURFMINMEM) 1
#set env(VMDQUICKSURFVERBOSE) 1
#set env(VMDOPTIXVERBOSE) DEBUG

#puts "Force CPU quicksurf"
#set env(VMDNOCUDA) 1

# Tachyon rendering flags
#render aasamples TachyonLOptiXInternal 144
#render aosamples TachyonLOptiXInternal 1

# OptiX rendering flags
#render aasamples TachyonLOptiXInternal 144
#render aosamples TachyonLOptiXInternal 1
#render aasamples TachyonLOptiXInternal 12
#render aosamples TachyonLOptiXInternal 12

#display backgroundgradient on
set scriptfiles { hiv-vr360.vmd hiv-vr360.vpt }
render_movie $scriptfiles 0 1 0 0  /ssd/johns/movies

quit