examples/video.py
This is one of the example scripts included
with Shady. These scripts can be run conventionally like any
normal Python script, or you can choose to run them as
interactive tutorials, for example with python -m Shady demo video
# $BEGIN_SHADY_LICENSE$
#
# This file is part of the Shady project, a Python framework for
# real-time manipulation of psychophysical stimuli for vision science.
#
# Copyright (c) 2017-2022 Jeremy Hill, Scott Mooney
#
# Shady is free software: you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see http://www.gnu.org/licenses/ .
#
# $END_SHADY_LICENSE$
#: Decoding and rendering of frames from a movie file or camera
"""
This demo script shows some of the things that can be done with
video stimuli.
By explicitly importing `Shady.Video`, you can enable the `.video`
property of the `Stimulus` class. You can assign the filename of
a video file, or the integer ID of a live camera, to this property
(it is actually redirected to `.video.source`, with `.video`
itself being an object that is implicitly constructed when needed).
Video file decoding and live camera acquisition both require the
`cv2` module from the third-party package`opencv-python`, which
can be installed via `pip`.
Be warned that video stimuli are fundamentally less efficient
than Shady's other more typical stimulus approaches: rather than
transferring everything to the graphics card up-front, or allowing
everything to be generated on-the-fly on the GPU, this uses
CPU code to decode every new frame from the video source and then
to transfer it from CPU to GPU. Timing performance will likely
suffer as a result.
"""#.
import random
if __name__ == '__main__':
import random
import Shady
"""
Parse command-line options:
"""#:
cmdline = Shady.WorldConstructorCommandLine( width=1000, height=750 )
source = cmdline.Option( 'source', Shady.PackagePath( 'examples/media/fish.mp4' ), type=( int, str ), container=None, doc="Supply the path-/file-name of a movie file, or an integer (starting at 0) to address a camera." )
loop = cmdline.Option( 'loop', True, type=bool, container=None, doc="Specifies whether or not to play the video on infinite loop." )
transform = cmdline.Option( 'transform', False, type=bool, container=None, doc="Demonstrate ways to use, and not to use, the .video.Transform callback" )
gauge = cmdline.Option( 'gauge', transform, type=bool, container=None, doc="Whether or not to show a `FrameIntervalGauge`." )
multi = cmdline.Option( 'multi', 0, type=int, container=None, doc="Specifies the number of animated copies of the video to make. Copying and animation is efficient (happens all on GPU)." )
cmdline.Help().Finalize()
"""
Enable video decoding by explicitly importing `Shady.Video`
then create a `World`:
"""#:
import Shady.Video
w = Shady.World( **cmdline.opts )
if gauge: Shady.FrameIntervalGauge( w )
"""
Create a stimulus and set its `.video` property:
"""#:
s = w.Stimulus( video=source, bgalpha=0 )
s.video.Play( loop=loop )
""#>
if transform:
"""
According to the command-line arguments, we've been asked
to demonstrate the `.video.Transform()` mechanism. So, let's
create a state machine. We will rotate, in 3-second steps,
between various options that will differently affect timing.
"""#:
sm = Shady.StateMachine()
"""
Some of the steps will use this helper function as the
video transform:
"""#:
def Desaturated( x ):
gray = x[ :, :, :3 ].mean( axis=2 )
x[ :, :, 0 ] = gray
x[ :, :, 1 ] = gray
x[ :, :, 2 ] = gray
return x
"""
In the first step, the video is not transformed. Shady
only transfers new data from CPU to GPU when a new image
is supplied by the camera or decoded from the file. Note
however, that even this will affect frame timing: Shady
operates most smoothly when all possible image frames
have been pre-loaded onto the GPU.
"""#:
@sm.AddState
class DoNothing( Shady.StateMachine.State ):
duration = 3
next = 'TransformNewData'
def onset( self ):
s.video.Transform = None
"""
The next 3-second state demonstrates an efficient video-frame
transformation: every time a *new* frame is decoded, transform
it (in this case, desaturate it). Not every display frame
will entail a new frame of video content however, so if the
video hasn't changed, return `None` to signal that nothing
needs to be done.
"""#:
@sm.AddState
class TransformNewData( Shady.StateMachine.State ):
duration = 3
next = 'ReturnOriginalEveryTime'
def onset( self ):
s.video.Transform = lambda x, changed: Desaturated( x ) if changed else None
"""
Add a 3-second state demonstrating an INEFFICIENT abuse of
the transformation mechanism. The transformation is just the
identity transform, so no change will be visible, but a
"transformed" image is returned on every display frame. This
means Shady will think it has to transfer a new texture to
the GPU on every display frame, which ordinarily it would not
do (normally it only needs to do this at the video frame rate,
not the display frame rate).
"""#:
@sm.AddState
class ReturnOriginalEveryTime( Shady.StateMachine.State ):
duration = 3
next = 'TransformEveryTime'
def onset( self ):
s.video.Transform = lambda x, changed: x
"""
Another example of what NOT to do: here we have the
desaturating transformation implemented INEFFICIENTLY because
it returns something on *every* display frame, regardless of
whether or not there is new video content.
"""#:
@sm.AddState
class TransformEveryTime( Shady.StateMachine.State ):
duration = 3
next = 'DoNothing'
def onset( self ):
s.video.Transform = lambda x, changed: Desaturated( x )
"""
Ensure `sm( t )` is called on every frame:
"""#:
s.SetAnimationCallback( sm )
"""
The following function will be used if the --multi option was
set >0. It illustrates how you can use an existing `Stimulus`
instance as the `source` of a new `Stimulus` during construction,
causing the new `Stimulus` to share the old one's `.textureID`.
"""#:
def Spawn( multi ):
s.plateauProportion = 1 # oval/circular
s.video.aperture = min( s.video.aperture ) # definitely circular
t0 = w.timeInSeconds # for synchronization (see below)
for i in range( multi ):
cyclical_offset = i / float( multi )
cycle = Shady.Integral( 0.2 ) + cyclical_offset
position = Shady.Apply( s.Place, cycle * 360, polar=True )
anchor = Shady.Apply( Shady.Sinusoid, cycle, phase_deg=[ 270, 180 ] )
# Each newly-created Integral starts its clock the first time it is called.
# We'll call them once below, explicitly, with t0, before using them
# as dynamic property values. This ensures that, even if each child stimulus
# takes time to create, the dynamic properties are in synch across copies.
# (This would be an issue in a threaded environment if you were to call
# `Spawn(multi)` directly rather than `w.Defer( Spawn, multi )` because in
# that case each `w.Stimulus()` call below would be deferred to the end of
# the current frame, with the engine waiting synchronously for each one
# to complete - so, it would take one frame per child).
# create a new Stimulus that shares the old one's texture:
child = w.Stimulus( s, position=position( t0 ), anchor=anchor( t0 ) )
# copy all the physical properties of the parent Stimulus:
child.Inherit( s )
# actually share some of the properties (texture and texture dimensions):
s.ShareTexture( 'envelopeSize', 'textureSize', child )
# individually scale and animate the copy:
child.Set( position=position, anchor=anchor, scaling=0.2 )
child.color = [ random.random() for channel in 'rgb' ]
if multi:
w.Defer( Spawn, multi ) # use of .Defer() means that the explicit t0 synchronization, above, isn't actually required
"""
Set an event handler for keyboard control of the video:
"""#:
@w.EventHandler( -1 )
def VideoKeyControl( self, event ):
if event.abbrev in 'kp[ ]':
s.video.playing = not s.video.playing
if event.abbrev in 'kp[left] ka[left]' and not s.video.live:
s.video.Pause() # another syntax for setting s.video.playing = False
if 'shift' in event.modifiers: s.video.frame = 0 # rewind to start
else: s.video.frame -= 1 # step back
if event.abbrev in 'kp[right] ka[right]' and not s.video.live:
s.video.Pause() # another syntax for setting s.video.playing = False
if 'shift' in event.modifiers: s.video.frame = -1 # skip to end
else: s.video.frame += 1 # step forward
""#>
print( """
space pause/unpause
left-/right-arrow step back/forward one frame
shift + left-/right-arrow rewind to first frame/skip ahead to last frame
""" )
Shady.AutoFinish( w )