From 848f87da4ea7e1039468e50626e2d090f3517510 Mon Sep 17 00:00:00 2001 From: Karl Kroening Date: Sun, 25 Nov 2018 03:47:49 -0600 Subject: [PATCH] Add numpy_stream example --- examples/graphs/tensorflow-stream.png | Bin 0 -> 6005 bytes examples/tensorflow_stream.py | 248 ++++++++++++++++++++++++++ 2 files changed, 248 insertions(+) create mode 100644 examples/graphs/tensorflow-stream.png create mode 100644 examples/tensorflow_stream.py diff --git a/examples/graphs/tensorflow-stream.png b/examples/graphs/tensorflow-stream.png new file mode 100644 index 0000000000000000000000000000000000000000..e5047a168e7b79742750bf3f176907cd6c994110 GIT binary patch literal 6005 zcmbtYX*`r|)K5i4BGFi~LaWzCjtY%|He4zio< zjIqqbU@(mL_B_vr_w)OHnK|eBUCVu)>wlL2iF&N3$w<#hf9ljJMy-eH2B%KZ909WD z`P0Dfs7j{YsZ$_jE%kdw{*xOSLC=jwCOc~BJB=S-U}a`z(_lU8PLp$*ZF*p_fac{a zx7KCwg-`V1+OJ=zVsEiAstBc~nSFY-$i~K0koparnln!mXOO>My*)zakl$GD{gC>m zxye2RjXA8?O{=!AaW0whQ*rKVNeZC9$9xw5iQSv#>^){A6QI1IF#n&$SqQN7ETL!6 z9ew1B>(AKR^XKB?S_UsIEgc<_FxJ-Ay>Nz|;8HlLPu}sG`*h>}Z1Wk$z7owio{jOgxE-DgL zRt_G>x>sFY%{kw5L?K5q@#OugoK2_ zvB?{0GOS>Sci>`^G9hGE!Uz{=^)BNF1`YQWYw>h(C=|-MNbOCPc6M?a7@TCa7=-5IhGl1A8CvCCo1J|PeG~l+6dHMn zyFPgThx*Yz3DPuB7_hAwlIZ-hSvLTOy+7bMb&b0aWy9= z-4qq=9v!{9CW$o7ovd-?vPg-Kk1s5AZ1CNvDV8BOgN&?{BuDCB(%`{D{*+LPAt36$G|iTs1T>D472BE7VTa zz<>n?MwS&V`h=*fx2D{-X*H)ich2xjA_C#&=s1Cb1tr@T6&Kso!SO@iE?g1v-k9_` z+}qrCoEKik;h3%|42+Mb(1}S%#NL8oW~@Dg(Df*3%}*(<0h+} zXM%`#)4W>W5z)tpbcyBVWm;NVTU%Q&YH$AUT+774&f3@-9zR&e#>SQ|=NdQ{4ochc z;_`xz6d8w(XCzPT%ft>#1774JO@9ji7g+N4^}xVm0fF2wjQXWlDsLXh$;tWlB#95V z{LXu{0LhFnx3bEdymmM6^MGpk?0V6jdfx|z=eHa{3EJA)1<#&5dBO$dh1a&>7vfa% zP*<4-Za6vIVq|2DVqL%fXHg_iDQNdr(8~IHZ|WB_gV!E&zi4+o0N`xj>FVlQzpjxi zZP)vv2e|jQT2V2v3-an8FAF^In4#`YxsyInn0O(qXUVp{Q~*Jp1;aMa$Kv;wHb~wp z1J;z?jrBQfotL8n($8_PH8P4>zx+jI5o@OZQa4-Q5x4B2*mF?k9j}O|JoC+vd00pv z`=+avSq~Gp;%6mqFE55mmy#wv zGwZ;g{jDrB@z=fZ;CGRYp+^D$AY)IgpJhVbLX!RouuHr z5?4#3(fi9>Vq(RBU)0-Iag!^g$LXZ=YHE_~dVhs85UGh4e-GBm;5YJmlEEg%#!s3Q zOh=Ycu{fgG%3-c%+>lCkdHKh8`s%2VqxvGOeWZwpNK_E{=bJM|l>ww>ySlhT7}E59 zM;ved&YRvTxh*oSiVcLaE`4y#ZRXVg61Tu@QAP_sYc?fIEAkNL* z_FQ~uVlvj*`4Fo5dor^J@$|JjeYdWzu4TXy8{rQSW)BW+I$~veV27m)py*-%i@@Ws zOeuZ&hz56ZsAb=K`I4XU5dS;>=mDYSh8RB7~t)mOWOYa3Km0&_Vd$2|9n$sYHAl`H)(VXD-yK3p3O-= zk+uYBP~h9Px3{0E_bEmq#|H=F8%uhlW+BgSl}=RJSIU2Cf=7Me8TPu3#6)x0*xG6| zD`>=XD|++EyfcjWoR;R~<~Chwp>*cV86_npf=OOgRn=gwrv4X&_}4#NCZwDMh)$X@ z?AJFB-OM6G!F|3|eU&5iVIHYh@6=Ns$*ABoifOC<>4&=|aV2|5d0sggZ-@gLR8yypKkzQ9XL$TT$Z@{^C zco=L6wR*R&%#vfPK)Zu=Y7!3z9z6w&)m$zUsse%}6BCm$X0kDW1i(Us({%>b6c(nY zt^HozY0L{3&9_Bn^A3T*?l!S;aZ#R2T->L4+@6Q6lr|%+xzP|Kr^YG~0M_;Yn`y5EeBl&fO@VCOjiVt?xjcWn^wQ&a-^=P- zua$>YM_ri#FdKXw*!*O{Jj{c4GX%&N!uu!7LGLTP?1F`BWzyJ{ ze7_?QmX99~067f4zm%MWk#vqk4W?$|^{63G=xy#@;*5%zm`_9=(>#3&K2|r7ra(M^V<*q|aLx78bJnHSmeB4S|7c=b19Pahp3kwn_t7 ztg@(RQGLCc79nVR(?TgoGaahpzj(+|cFVrUtP2JOB_}3A^}{&iTuiO4uZ#Da{W^N; zR~xElNSHTY{SM(lRgMuHdw)qnA{1*we+RJ6d@|8D^iCbK$y#uQ4-x;nji#9;L z=@#ZNDQ1#3{Hfb5x@hZ|IziIp7MiP9--d2avfVDm{!{{No#u_y>3rpV+ zdEvr^`8eGiRh#}VcYs{|-ae(6oU-rOaBeR~TCVc%#gtF5b(7K%E$U@j#!*5s{;tE@ z_`~+LdCS0VX$tgAiI zHK1npm~Q)8h%!Y!8}N?CTacUHCYCiW69C-OeJXTkpoFBc7aNc~GGBYA(JQCYA$QIz zh^$z(M5I55-81-XeX$AOoziq0be%{8GEi8)>}_0s#ZpMi=_GeWMQZ}*-LTl@q(3Da z86=GIr*dEKcF2$X!tkY(&b{rH;~c|+CFh>Xv}(BS?sZ&quG)epEDVR&ZUm!PzIxg_ z+XbGk^i;~V|9K;^)4|12EKRjXKB|4B>qAtR^>UL~nr=XTI_J2;Z3a2XiaK4BbhJGt zq9v!+;L>|YqcKuH`)ycQ*tRsOv8gHg{rknf6Qs-Ol`_b!f9Si59choxm8-=Fl+pC{ z6zWJhJW~zt{F;-8|72M2q?zbJB6lSWaXUCW_e>kwdLEmO!Vkvl+9H>@^^QT@ECI>7 zeS3-|9=1>X(Z~X9y;s~@JnfVT;#<19){tK!t$v{Cj)dufj|PF4GIp%)TF_qCk!7OU zD>60km(a+fc4lF|mfDr!=8XS*bw>n7`$xx;M`}YBtzje->V-`XP?Sxl4jBZ>3+EYG zVP13>&>BrQ&cC(Kj6dByQ&v(UD=VAG&T94t;7h(h0;hj|>$l&Rh9y~#otJqMm&F+h zm{Ye+-X*nspUnQ}skHK0jUf0rWrx;cFS*aMy+O6DKL_?o=HWlwQ+-0_`k3M>ZFS7^ zH72ZHaZ}iW9QRTw>tBqM-`G^lMVOjGlZR@peU_|pmn~XEdUyon8 z!!#cFO9n<8#FUksPby$OW+!;p+k05aH;eIm>Nd7Q;(CAM#qE{+6u*LVS`dFM3UqOv zAb~y@P1NbMU%&3tN+jWxpadQHPDLTl&D9LuEgm5Uo(1l&!w~%*3Ckf7^?)KP^s(^n zz=YRl_`?`wf?;#BZFw(@$CD6a%v)~jxyHrC^!N?`fxSb0_D?D+wUT%IJr<-&MV%$r z{ZNBXU~!!6X_A{7Irsacj^`m`eEd&XUp{`56BGNO_b)UjIakbPBrLTmYlj>lq>p(1 z^l7`Rp$vuMsy_d;eqYDSs-EVNf|-TlE^jv~f-b`89u>6>iObKo2W$(2HO(KvVkr~R zs4Nc@dg1Xdn}Uo{AP9wSVNRl`I~CRbDF622%O&|+MD=m7F^216KNJ0Sa{p&3S!5ZzG!Fg%j-P%hQz59Bhlp)5e zw$VvDu}A5aYG=&Tp!uzBI*Q>r?gEA+G47!Jte@Tu)HnP1{mMQiZGoE7nKyvQiFzmd2+Ra;pGbK2d1PqD=Z@Vo zbMvxil@It!3^@pKQ873-qEsD5*0k=(g|f?S!1QM{?+^yiS~m_J7MA<9x`OaAK|}=(g zv9E0NCu&~2#(U+9cK-RB@XQBqbWB%9ia{mMZ3S=1EWGUZ+d`h4o#8lx(7HS9gkBSc zZhq2>n&Ri|X0&BopF2gY5D3G0VXbQLT%wB^)Jj8;b1bFuQ?z+MZ1hi6Xq;(fI4ofA z+;1nFC!;vUc=XIIa=tj5>xvlWD7hsyR9lF2-ex8)HB)roCn3k&8m@Bl<~uW%y7$r1 zf}V6~OVpmZ9(vKtS3LW*Fcr*xVdg^s6!)m9nty`o2ewE@3FzN*?OP9azb`f&Cp z=!QA`sq6Su$N@w(U-Z#kTH>>4hzWwSrFW9KQY$ouTm1^IDcPP$tb5Lvy$EGdbX+Qu zy!Wgo+V(*23xJF*%)70%L$wsL{`*VvN%9$qjmcVK#=>!iQPIiqsuL6S9m9eguj*_4 z;QAFRn$w6g5$GCav9)n{qmcRvo~s-ytQeP(_%j3Iif;#UZ6Xe(OQ|f=8g%yYgIlFh zHKWxBxuiLlPCBp)VGCYjT~*k2bW_@w*T4) zfsOjIwXGU2R(>B3&e4&ll*+{(aDQmWJnI|51mXnvk%i&5{`arfVy+~>I=@G=X&kw5 zrCXk96z(@dP^A8dOA<(bwL;dFdrFvX&;R{1BJ)8(A%O)t(N zbfZ9gT34v~0h2RuIW3w7POXE!NnXs1`(<(fB^Y6MEKyrc^#EaJI!NTHp|-w*Qb08s zz~CIC8ZHyX0>!&*hR};(a`GZf!#X^4(6xpJfsVrwNs`mUE&-_4s$#gLhwf7P1xa6ZYn$ zv{r0|@tjoPoimzL)K!2Ofh-%r=c})hX%Cp@^c%|KHgCXr=W)dP|D~QLXUhR>mI-yjqaN1UF(BhO&Y)gA=kq`(O40Cf@AjhC2-oaRH=!;bwNfGgZMp$clV5PsTT_Ew7Pjej*?y6VTBGq`+!+d1~P&7w*f z_9^u>ms$Wy^sh1t`RWX`-zRz>i_gp|old7ad((f{V1c?|3dA?vqwL1by&Bf}HU{&-Mst zhoo__B27CizW4LI)YH2pe!ngAiiM@6-VV?ZN%9YoS>1n*_QL5sQsb@E7ztc0?(Xi+ yPXo1UBwO? [ffmpeg process 1] -> [python] -> [ffmpeg process 2] -> (output video) + +This example reads/writes video files on the local filesystem, but the +same pattern can be used for other kinds of input/output (e.g. webcam, +rtmp, etc.). + +The simplest processing example simply darkens each frame by +multiplying the frame's numpy array by a constant value; see +``process_frame_simple``. + +A more sophisticated example processes each frame with tensorflow using +the "deep dream" tensorflow tutorial; activate this mode by calling +the script with the optional `--dream` argument. (Make sure tensorflow +is installed before running) +''' +from __future__ import print_function +import argparse +import ffmpeg +import logging +import numpy as np +import os +import subprocess +import zipfile + + +parser = argparse.ArgumentParser(description='Example streaming ffmpeg numpy processing') +parser.add_argument('in_filename', help='Input filename') +parser.add_argument('out_filename', help='Output filename') +parser.add_argument( + '--dream', action='store_true', help='Use DeepDream frame processing (requires tensorflow)') + +logger = logging.getLogger(__name__) +logging.basicConfig(level=logging.INFO) + + +def get_video_size(filename): + logger.info('Getting video size for {!r}'.format(filename)) + probe = ffmpeg.probe(filename) + video_info = next(s for s in probe['streams'] if s['codec_type'] == 'video') + width = int(video_info['width']) + height = int(video_info['height']) + return width, height + + +def start_ffmpeg_process1(in_filename): + logger.info('Starting ffmpeg process1') + args = ( + ffmpeg + .input(in_filename) + .output('pipe:', format='rawvideo', pix_fmt='rgb24', vframes=8) + .compile() + ) + return subprocess.Popen(args, stdout=subprocess.PIPE) + + +def start_ffmpeg_process2(out_filename, width, height): + logger.info('Starting ffmpeg process2') + args = ( + ffmpeg + .input('pipe:', format='rawvideo', pix_fmt='rgb24', s='{}x{}'.format(width, height)) + .output(out_filename, pix_fmt='yuv420p') + .overwrite_output() + .compile() + ) + return subprocess.Popen(args, stdin=subprocess.PIPE) + + +def read_frame(process1, width, height): + logger.debug('Reading frame') + + # Note: RGB24 == 3 bytes per pixel. + frame_size = width * height * 3 + in_bytes = process1.stdout.read(frame_size) + if len(in_bytes) == 0: + frame = None + else: + assert len(in_bytes) == frame_size + frame = ( + np + .frombuffer(in_bytes, np.uint8) + .reshape([height, width, 3]) + ) + return frame + + +def process_frame_simple(frame): + '''Simple processing example: darken frame.''' + return frame * 0.3 + + +def write_frame(process2, frame): + logger.debug('Writing frame') + process2.stdin.write( + frame + .astype(np.uint8) + .tobytes() + ) + + +def run(in_filename, out_filename, process_frame): + width, height = get_video_size(in_filename) + process1 = start_ffmpeg_process1(in_filename) + process2 = start_ffmpeg_process2(out_filename, width, height) + while True: + frame = read_frame(process1, width, height) + if frame is None: + logger.info('End of input stream') + break + + logger.debug('Processing frame') + frame = process_frame(frame) + write_frame(process2, frame) + + logger.info('Waiting for ffmpeg process1') + process1.wait() + + logger.info('Waiting for ffmpeg process2') + process2.stdin.close() + process2.wait() + + logger.info('Done') + + +class DeepDream(object): + '''DeepDream implementation, adapted from official tensorflow deepdream tutorial: + https://github.com/tensorflow/tensorflow/tree/master/tensorflow/examples/tutorials/deepdream + + Credit: Alexander Mordvintsev + ''' + + _DOWNLOAD_URL = 'https://storage.googleapis.com/download.tensorflow.org/models/inception5h.zip' + _ZIP_FILENAME = 'deepdream_model.zip' + _MODEL_FILENAME = 'tensorflow_inception_graph.pb' + + @staticmethod + def _download_model(): + logger.info('Downloading deepdream model...') + try: + from urllib.request import urlretrieve # python 3 + except ImportError: + from urllib import urlretrieve # python 2 + urlretrieve(DeepDream._DOWNLOAD_URL, DeepDream._ZIP_FILENAME) + + logger.info('Extracting deepdream model...') + zipfile.ZipFile(DeepDream._ZIP_FILENAME, 'r').extractall('.') + + @staticmethod + def _tffunc(*argtypes): + '''Helper that transforms TF-graph generating function into a regular one. + See `_resize` function below. + ''' + placeholders = list(map(tf.placeholder, argtypes)) + def wrap(f): + out = f(*placeholders) + def wrapper(*args, **kw): + return out.eval(dict(zip(placeholders, args)), session=kw.get('session')) + return wrapper + return wrap + + @staticmethod + def _base_resize(img, size): + '''Helper function that uses TF to resize an image''' + img = tf.expand_dims(img, 0) + return tf.image.resize_bilinear(img, size)[0,:,:,:] + + def __init__(self): + if not os.path.exists(DeepDream._MODEL_FILENAME): + self._download_model() + + self._graph = tf.Graph() + self._session = tf.InteractiveSession(graph=self._graph) + self._resize = self._tffunc(np.float32, np.int32)(self._base_resize) + with tf.gfile.FastGFile(DeepDream._MODEL_FILENAME, 'rb') as f: + graph_def = tf.GraphDef() + graph_def.ParseFromString(f.read()) + self._t_input = tf.placeholder(np.float32, name='input') # define the input tensor + imagenet_mean = 117.0 + t_preprocessed = tf.expand_dims(self._t_input-imagenet_mean, 0) + tf.import_graph_def(graph_def, {'input':t_preprocessed}) + + self.t_obj = self.T('mixed4d_3x3_bottleneck_pre_relu')[:,:,:,139] + #self.t_obj = tf.square(self.T('mixed4c')) + + def T(self, layer_name): + '''Helper for getting layer output tensor''' + return self._graph.get_tensor_by_name('import/%s:0'%layer_name) + + def _calc_grad_tiled(self, img, t_grad, tile_size=512): + '''Compute the value of tensor t_grad over the image in a tiled way. + Random shifts are applied to the image to blur tile boundaries over + multiple iterations.''' + sz = tile_size + h, w = img.shape[:2] + sx, sy = np.random.randint(sz, size=2) + img_shift = np.roll(np.roll(img, sx, 1), sy, 0) + grad = np.zeros_like(img) + for y in range(0, max(h-sz//2, sz),sz): + for x in range(0, max(w-sz//2, sz),sz): + sub = img_shift[y:y+sz,x:x+sz] + g = self._session.run(t_grad, {self._t_input:sub}) + grad[y:y+sz,x:x+sz] = g + return np.roll(np.roll(grad, -sx, 1), -sy, 0) + + def process_frame(self, frame, iter_n=10, step=1.5, octave_n=4, octave_scale=1.4): + t_score = tf.reduce_mean(self.t_obj) # defining the optimization objective + t_grad = tf.gradients(t_score, self._t_input)[0] # behold the power of automatic differentiation! + + # split the image into a number of octaves + img = frame + octaves = [] + for i in range(octave_n-1): + hw = img.shape[:2] + lo = self._resize(img, np.int32(np.float32(hw)/octave_scale)) + hi = img-self._resize(lo, hw) + img = lo + octaves.append(hi) + + # generate details octave by octave + for octave in range(octave_n): + if octave>0: + hi = octaves[-octave] + img = self._resize(img, hi.shape[:2])+hi + for i in range(iter_n): + g = self._calc_grad_tiled(img, t_grad) + img += g*(step / (np.abs(g).mean()+1e-7)) + #print('.',end = ' ') + return img + + +if __name__ == '__main__': + args = parser.parse_args() + if args.dream: + import tensorflow as tf + process_frame = DeepDream().process_frame + else: + process_frame = process_frame_simple + run(args.in_filename, args.out_filename, process_frame)