Commit f2bf10a6 authored by Breno Rilho Lemos's avatar Breno Rilho Lemos 💬
Browse files

Make Blender scenes in parallel using GNU parallel

parent 0892519c
...@@ -88,7 +88,7 @@ TPC=1 ...@@ -88,7 +88,7 @@ TPC=1
DETAILED_TPC=0 DETAILED_TPC=0
TRD=1 TRD=1
EMCAL=1 EMCAL=1
BLENDERSAVE=1 BLENDERSAVE=0
PICPCT=80 PICPCT=80
# now enjoy the options in order and nicely split until we see -- # now enjoy the options in order and nicely split until we see --
while true; do while true; do
...@@ -277,7 +277,7 @@ if [[ $CAMERAS != "" ]]; then ...@@ -277,7 +277,7 @@ if [[ $CAMERAS != "" ]]; then
CAMERAS=$(echo $CAMERAS | sed -e 's#,#Camera #g' -e 's#$#Camera#') CAMERAS=$(echo $CAMERAS | sed -e 's#,#Camera #g' -e 's#$#Camera#')
fi fi
if [[ $MOSAIC == "true" ]]; then if [[ $MOSAIC = "true" ]]; then
CAMERAS=$(echo "OverviewCamera BarrelCamera SideCamera ForwardCamera") CAMERAS=$(echo "OverviewCamera BarrelCamera SideCamera ForwardCamera")
fi fi
...@@ -386,7 +386,7 @@ if [ "$SAMPLE" = "true" ]; then ...@@ -386,7 +386,7 @@ if [ "$SAMPLE" = "true" ]; then
-fps=${FPS} -resolution=${RESOLUTION} -transparency=${TRANSPARENCY} \ -fps=${FPS} -resolution=${RESOLUTION} -transparency=${TRANSPARENCY} \
-stamp_note="opendata.cern.ch_record_1102_alice_2010_LHC10h_000139038_ESD_0001_2" -its=${ITS}\ -stamp_note="opendata.cern.ch_record_1102_alice_2010_LHC10h_000139038_ESD_0001_2" -its=${ITS}\
-tpc=${TPC} -trd=${TRD} -emcal=${EMCAL} -detailed_tpc=${DETAILED_TPC} \ -tpc=${TPC} -trd=${TRD} -emcal=${EMCAL} -detailed_tpc=${DETAILED_TPC} \
-blendersave=${BLENDERSAVE} -picpct=${PICPCT} -tpc_blender_path=${BLENDER_SCRIPT_DIR} \ -blendersave=1 -picpct=${PICPCT} -tpc_blender_path=${BLENDER_SCRIPT_DIR} \
-output_path="${BLENDER_OUTPUT}" -output_path="${BLENDER_OUTPUT}"
popd popd
...@@ -515,147 +515,216 @@ elif [ "$SAMPLE" = "false" ]; then ...@@ -515,147 +515,216 @@ elif [ "$SAMPLE" = "false" ]; then
EXTRACTED_FILES=$(ls -1 ${UNIQUEID}_*.dat | sort --version-sort) EXTRACTED_FILES=$(ls -1 ${UNIQUEID}_*.dat | sort --version-sort)
for LOCAL_FILE_WITH_DATA in $EXTRACTED_FILES; do if ! grep -q "${UNIQUEID}, REMOVED UNUSED FILES" $PROGRESS_LOG; then
pushd ${BLENDER_SCRIPT_DIR} for LOCAL_FILE_WITH_DATA in $EXTRACTED_FILES; do
EVENT_ID=$(echo $LOCAL_FILE_WITH_DATA | \ ##############################
sed -e "s#${UNIQUEID}_##" \ # Phase 2: Event selection #
-e "s#\.dat##") ##############################
EVENT_UNIQUE_ID=${UNIQUEID}_${EVENT_ID}
############################## NUMBER_OF_PARTICLES=$(wc -l $LOCAL_FILE_WITH_DATA | \
# Phase 2: blender animate # awk '{ print $1 }')
##############################
NUMBER_OF_PARTICLES=$(wc -l $LOCAL_FILE_WITH_DATA | \ AVERAGE_PZ=$(awk 'BEGIN {pzsum=0;n=0} {pzsum+=$8;n++} END {print sqrt(pzsum*pzsum/n/n)}'\
awk '{ print $1 }') ${LOCAL_FILE_WITH_DATA})
AVERAGE_PT=$(awk 'BEGIN {ptsum=0;n=0} {ptsum+=$9;n++} END {print ptsum/n}' \
${LOCAL_FILE_WITH_DATA})
echo "File $LOCAL_FILE_WITH_DATA:"
echo "Number of particles: $NUMBER_OF_PARTICLES"
echo "Average Z momentum: $AVERAGE_PZ"
echo "Average transversal momentum $AVERAGE_PT"
EVENT_COUNTER=$(more event_counter.txt)
######################################################################
# Remove text data of files for events that will not be animated: #
######################################################################
if (( $(echo "$AVERAGE_PT < $MIN_AVG_PT" |bc -l) )); then
echo "Average Transversal Momentum too low (minimum accepted is $MIN_AVG_PT). Deleting data text file."
# Remove non-processed files
rm -f $LOCAL_FILE_WITH_DATA
elif (( $(echo "$AVERAGE_PZ < $MIN_AVG_PZ" |bc -l) )); then
AVERAGE_PZ=$(awk 'BEGIN {pzsum=0;n=0} {pzsum+=$8;n++} END {print sqrt(pzsum*pzsum/n/n)}'\ echo "Average Z Momentum too low (minimum accepted is $MIN_AVG_PZ). Deleting data text file."
${LOCAL_FILE_WITH_DATA})
AVERAGE_PT=$(awk 'BEGIN {ptsum=0;n=0} {ptsum+=$9;n++} END {print ptsum/n}' \ # Remove non-processed files
${LOCAL_FILE_WITH_DATA}) rm -f $LOCAL_FILE_WITH_DATA
elif [[ $NUMBER_OF_PARTICLES -lt $MIN_PARTICLES ]]; then
echo "Too little particles (minimum accepted is $MIN_PARTICLES). Deleting data text file."
# Remove non-processed files
rm -f $LOCAL_FILE_WITH_DATA
elif [[ $NUMBER_OF_PARTICLES -gt $MAX_PARTICLES ]]; then
echo "Too many particles (maximum accepted is $MAX_PARTICLES). Deleting data text file."
# Remove non-processed files
rm -f $LOCAL_FILE_WITH_DATA
elif [[ $EVENT_COUNTER -ge $N_OF_EVENTS ]]; then
echo "Numbers of events set to be animated has already been reached. Deleting data text file."
# Remove non-processed files
rm -f $LOCAL_FILE_WITH_DATA
else
echo "File $LOCAL_FILE_WITH_DATA has $NUMBER_OF_PARTICLES particles." # Increment event counter
echo "Average Z momentum: $AVERAGE_PZ" EVENT_COUNTER=$EVENT_COUNTER+1
echo "Average transversal momentum $AVERAGE_PT" rm -f event_counter.txt
echo "$EVENT_COUNTER" > event_counter.txt
EVENT_COUNTER=$(more event_counter.txt) echo "This event will be animated. Continue."
if (( $(echo "$AVERAGE_PT >= $MIN_AVG_PT" |bc -l) )); then fi
if (( $(echo "$AVERAGE_PZ >= $MIN_AVG_PZ" |bc -l) )); then
if [[ $NUMBER_OF_PARTICLES -le $MAX_PARTICLES && $NUMBER_OF_PARTICLES \
-ge $MIN_PARTICLES && $EVENT_COUNTER -lt $N_OF_EVENTS ]]; then
if ! grep -q "${UNIQUEID}, ${EVENT_ID}, SCENE, FINISHED" $PROGRESS_LOG; then echo " " # Skip line to make it neat
echo "Processing ${EVENT_UNIQUE_ID} ($NUMBER_OF_PARTICLES tracks) in Blender" done
echo "Processing ${EVENT_UNIQUE_ID} in Blender" timestamp "${UNIQUEID}, REMOVED UNUSED FILES" >> $PROGRESS_LOG
timestamp "${UNIQUEID}, ${EVENT_ID}, SCENE, STARTING, $NUMBER_OF_PARTICLES" >> $PROGRESS_LOG fi
blender -noaudio --background -P animate_particles.py -- -radius=0.05 \
-duration=${DURATION} -datafile="${LOCAL_FILE_WITH_DATA}"\
-n_event=${EVENT_ID} -simulated_t=0.03 -fps=${FPS} -resolution=${RESOLUTION}\
-transparency=${TRANSPARENCY} -stamp_note="${EVENT_UNIQUE_ID}" -its=${ITS}\
-tpc=${TPC} -trd=${TRD} -emcal=${EMCAL} -detailed_tpc=${DETAILED_TPC} \
-blendersave=${BLENDERSAVE} -picpct=${PICPCT} -tpc_blender_path=${BLENDER_SCRIPT_DIR}\
-output_path="${BLENDER_OUTPUT}"
timestamp "${UNIQUEID}, ${EVENT_ID}, SCENE, FINISHED, $NUMBER_OF_PARTICLES" >> $PROGRESS_LOG
fi # Remove event counter file
rm -f event_counter.txt
# Now the list of extracted files shall only include events we want to animate
EXTRACTED_FILES=$(ls -1 ${UNIQUEID}_*.dat | sort --version-sort)
for type in $CAMERAS; do ################################################
# Create script so we can use GNU parallel #
# to create multiple Blender scenes #
################################################
if ! grep -q "${UNIQUEID}, PREPARED TO MAKE SCENES IN PARALLEL" $PROGRESS_LOG; then
rm -f scene-making
for LOCAL_FILE_WITH_DATA in $EXTRACTED_FILES; do
EVENT_ID=$(echo $LOCAL_FILE_WITH_DATA | \
sed -e "s#${UNIQUEID}_##" \
-e "s#\.dat##")
EVENT_UNIQUE_ID=${UNIQUEID}_${EVENT_ID}
# echo 'blender animate_particles.py' > scene-making
echo blender -noaudio --background -P animate_particles.py -- -radius=0.05 \
-duration=${DURATION} -datafile=\'${LOCAL_FILE_WITH_DATA}\' \
-n_event=${EVENT_ID} -simulated_t=0.03 -fps=${FPS} -resolution=${RESOLUTION} \
-transparency=${TRANSPARENCY} -stamp_note=\'${EVENT_UNIQUE_ID}\' -its=${ITS} \
-tpc=${TPC} -trd=${TRD} -emcal=${EMCAL} -detailed_tpc=${DETAILED_TPC} \
-blendersave=1 -picpct=${PICPCT} -tpc_blender_path=${BLENDER_SCRIPT_DIR} \
-output_path=\'${BLENDER_OUTPUT}\' >> scene-making
done
if ! grep -q "${UNIQUEID}, ${EVENT_ID}, ${type}, FINISHED" $PROGRESS_LOG; then timestamp "${UNIQUEID}, PREPARED TO MAKE SCENES IN PARALLEL" >> $PROGRESS_LOG
timestamp "${UNIQUEID}, ${EVENT_ID}, ${type}, STARTING, $NUMBER_OF_PARTICLES" >> $PROGRESS_LOG
blender -noaudio --background -P render.py -- -cam ${type} -datafile\
"${LOCAL_FILE_WITH_DATA}" -n_event ${EVENT_ID} -pic_pct ${PICPCT} -output_path "${BLENDER_OUTPUT}"
timestamp "${UNIQUEID}, ${EVENT_ID}, ${type}, FINISHED, $NUMBER_OF_PARTICLES" >> $PROGRESS_LOG fi
fi
done ###################################
# Make Blender scenes in parallel #
###################################
if ! grep -q "${UNIQUEID}, PARALLEL, SCENES, FINISHED" $PROGRESS_LOG; then
if [ "$MOSAIC" = "true" ]; then parallel --jobs $N_OF_EVENTS < scene-making
if ! grep -q "${UNIQUEID}, ${EVENT_ID}, MOSAIC, FINISHED" $PROGRESS_LOG; then timestamp "${UNIQUEID}, PARALLEL, SCENES, FINISHED" >> $PROGRESS_LOG
pushd ${BLENDER_OUTPUT}
timestamp "${UNIQUEID}, ${EVENT_ID}, MOSAIC, STARTING, $NUMBER_OF_PARTICLES" >> $PROGRESS_LOG fi
# Delete existing incomplete .mp4 file rm -f scene-making
if [[ -f ${EVENT_UNIQUE_ID}_Mosaic.mp4 ]]; then
rm ${EVENT_UNIQUE_ID}_Mosaic.mp4
fi
# Setting input names for clips in order to make mosaic clip #####################################
INPUT_ONE=$(ls *$EVENT_UNIQUE_ID*${FPS_DUR}.mp4 | awk 'NR==1') # Render scenes in selected cameras #
INPUT_TWO=$(ls *$EVENT_UNIQUE_ID*${FPS_DUR}.mp4 | awk 'NR==2') #####################################
INPUT_THREE=$(ls *$EVENT_UNIQUE_ID*${FPS_DUR}.mp4 | awk 'NR==3') for LOCAL_FILE_WITH_DATA in $EXTRACTED_FILES; do
INPUT_FOUR=$(ls *$EVENT_UNIQUE_ID*${FPS_DUR}.mp4 | awk 'NR==4')
ffmpeg -i ${INPUT_FOUR} -i ${INPUT_TWO} -i ${INPUT_THREE} -i ${INPUT_ONE} -filter_complex\ EVENT_ID=$(echo $LOCAL_FILE_WITH_DATA | \
"[0:v][1:v]hstack=inputs=2[top];[2:v][3:v]hstack=inputs=2[bottom];[top][bottom]vstack=inputs=2[v]"\ sed -e "s#${UNIQUEID}_##" \
-map "[v]" ${EVENT_UNIQUE_ID}_Mosaic.mp4 -e "s#\.dat##")
EVENT_UNIQUE_ID=${UNIQUEID}_${EVENT_ID}
NUMBER_OF_PARTICLES=$(wc -l $LOCAL_FILE_WITH_DATA | \
awk '{ print $1 }')
timestamp "${UNIQUEID}, ${EVENT_ID}, MOSAIC, FINISHED, $NUMBER_OF_PARTICLES" >> $PROGRESS_LOG for type in $CAMERAS; do
popd if ! grep -q "${UNIQUEID}, ${EVENT_ID}, ${type}, FINISHED" $PROGRESS_LOG; then
fi
fi timestamp "${UNIQUEID}, ${EVENT_ID}, ${type}, STARTING, $NUMBER_OF_PARTICLES" >> $PROGRESS_LOG
blender -noaudio --background -P render.py -- -cam ${type} -datafile\
"${LOCAL_FILE_WITH_DATA}" -n_event ${EVENT_ID} -pic_pct ${PICPCT} -output_path "${BLENDER_OUTPUT}"
if ! grep -q "${UNIQUEID}, ${EVENT_ID}, TEXT DATA MOVED to final location" $PROGRESS_LOG; then timestamp "${UNIQUEID}, ${EVENT_ID}, ${type}, FINISHED, $NUMBER_OF_PARTICLES" >> $PROGRESS_LOG
# Move processed file to final location
mv $LOCAL_FILE_WITH_DATA ${BLENDER_OUTPUT}/$LOCAL_FILE_WITH_DATA
timestamp "${UNIQUEID}, ${EVENT_ID}, TEXT DATA MOVED to final location" >> $PROGRESS_LOG
fi
echo "EVENT ${EVENT_UNIQUE_ID} DONE with FILE $LOCAL_FILE_WITH_DATA." fi
# Increment event counter done
EVENT_COUNTER=$EVENT_COUNTER+1
rm -f event_counter.txt
echo "$EVENT_COUNTER" > event_counter.txt
else ########################
# Make mosaic 2x2 clip #
########################
if [ "$MOSAIC" = "true" ]; then
if [[ $NUMBER_OF_PARTICLES -lt $MIN_PARTICLES ]]; then if ! grep -q "${UNIQUEID}, ${EVENT_ID}, MOSAIC, FINISHED" $PROGRESS_LOG; then
echo "Too little particles (minimum accepted is $MIN_PARTICLES). Continue." pushd ${BLENDER_OUTPUT}
elif [[ $NUMBER_OF_PARTICLES -gt $MAX_PARTICLES ]]; then
echo "Too many particles (maximum accepted is $MAX_PARTICLES). Continue."
elif [[ $EVENT_COUNTER -ge $N_OF_EVENTS ]]; then
echo "Numbers of events set to be animated has already been reached. Continue."
fi
# Remove non-processed files timestamp "${UNIQUEID}, ${EVENT_ID}, MOSAIC, STARTING, $NUMBER_OF_PARTICLES" >> $PROGRESS_LOG
rm -f $LOCAL_FILE_WITH_DATA
# Delete existing incomplete .mp4 file
if [[ -f ${EVENT_UNIQUE_ID}_Mosaic.mp4 ]]; then
rm ${EVENT_UNIQUE_ID}_Mosaic.mp4
fi fi
else
echo "Average Z Momentum too low (minimum accepted is $MIN_AVG_PZ). Continue."
# Remove non-processed files # Setting input names for clips in order to make mosaic clip
rm -f $LOCAL_FILE_WITH_DATA INPUT_ONE=$(ls *$EVENT_UNIQUE_ID*${FPS_DUR}.mp4 | awk 'NR==1')
INPUT_TWO=$(ls *$EVENT_UNIQUE_ID*${FPS_DUR}.mp4 | awk 'NR==2')
INPUT_THREE=$(ls *$EVENT_UNIQUE_ID*${FPS_DUR}.mp4 | awk 'NR==3')
INPUT_FOUR=$(ls *$EVENT_UNIQUE_ID*${FPS_DUR}.mp4 | awk 'NR==4')
ffmpeg -i ${INPUT_FOUR} -i ${INPUT_TWO} -i ${INPUT_THREE} -i ${INPUT_ONE} -filter_complex\
"[0:v][1:v]hstack=inputs=2[top];[2:v][3:v]hstack=inputs=2[bottom];[top][bottom]vstack=inputs=2[v]"\
-map "[v]" ${EVENT_UNIQUE_ID}_Mosaic.mp4
timestamp "${UNIQUEID}, ${EVENT_ID}, MOSAIC, FINISHED, $NUMBER_OF_PARTICLES" >> $PROGRESS_LOG
popd
fi fi
else
echo "Average Transversal Momentum too low (minimum accepted is $MIN_AVG_PT). Continue."
# Remove non-processed files
rm -f $LOCAL_FILE_WITH_DATA
fi fi
popd #######################################################
# Move text data files to where animations are stored #
#######################################################
if ! grep -q "${UNIQUEID}, ${EVENT_ID}, TEXT DATA MOVED to final location" $PROGRESS_LOG; then
done # Move processed file to final location
mv $LOCAL_FILE_WITH_DATA ${BLENDER_OUTPUT}/$LOCAL_FILE_WITH_DATA
timestamp "${UNIQUEID}, ${EVENT_ID}, TEXT DATA MOVED to final location" >> $PROGRESS_LOG
# Remove event counter file fi
pushd ${BLENDER_SCRIPT_DIR}
rm -f event_counter.txt echo "EVENT ${EVENT_UNIQUE_ID} DONE with FILE $LOCAL_FILE_WITH_DATA."
done
popd popd
########################
# Remove blender files #
########################
if [[ $BLENDERSAVE = 0 ]]; then
pushd ${BLENDER_OUTPUT}
rm -f *.blend
popd
fi
fi fi
timestamp "${UNIQUEID}, JOB FINISHED" >> $PROGRESS_LOG timestamp "${UNIQUEID}, JOB FINISHED" >> $PROGRESS_LOG
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment