From acae0e0988a1c20fb967e638b674369568a68e18 Mon Sep 17 00:00:00 2001 From: Carl Johnson Date: Wed, 22 Apr 2020 09:34:15 +0100 Subject: [PATCH] Stream switching (#160) --- README.md | 5 ++++- hooks/pre-checkout | 11 ++++++++--- python/fixture/server.zip | Bin 882289 -> 890977 bytes python/test_perforce.py | 29 ++++++++++++++++++++++++++++- 4 files changed, 40 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 16c115b..5da0395 100644 --- a/README.md +++ b/README.md @@ -75,7 +75,7 @@ steps: parallel: 16 ``` -### Share a stream workspace between pipelines. +### Share a stream workspace between pipelines Useful to avoid syncing duplicate data with large workspaces. Only allowed when there is a single buildkite agent running on the machine. @@ -85,7 +85,10 @@ steps: plugins: - improbable-eng/perforce: stream: //dev/buildkite + # Sync each stream once share_workspace: true + # Sync once and switch streams in-place (requires share_workspace: true) + stream_switching: true ``` ## Triggering Builds diff --git a/hooks/pre-checkout b/hooks/pre-checkout index 6c9ecf0..e5b26ec 100755 --- a/hooks/pre-checkout +++ b/hooks/pre-checkout @@ -14,9 +14,14 @@ if [[ "${BUILDKITE_PLUGIN_PERFORCE_SHARE_WORKSPACE}" == true ]] ; then echo "Error: You cannot share stream workspaces when running more than one agent" >&2 exit 1 fi - # Sanitize //depot/stream-name to __depot_stream-name - SANITIZED_STREAM=$(echo $STREAM | python -c "import sys; print(sys.stdin.read().replace('/', '_'));") - + if [[ "${BUILDKITE_PLUGIN_PERFORCE_STREAM_SWITCHING}" == true ]] ; then + echo "Stream switching enabled" + # Sanitize '//depot/stream-name' to 'depot' + SANITIZED_STREAM=$(echo $STREAM | python -c "import sys; print(sys.stdin.read().split('/')[2]);") + else # Create a directory per-stream + # Sanitize '//depot/stream-name' to '__depot_stream-name' + SANITIZED_STREAM=$(echo $STREAM | python -c "import sys; print(sys.stdin.read().replace('/', '_'));") + fi # Instead of builds//, checkout to builds/ PERFORCE_CHECKOUT_PATH="${BUILDKITE_BUILD_CHECKOUT_PATH}/../../${SANITIZED_STREAM}" export BUILDKITE_BUILD_CHECKOUT_PATH="${PERFORCE_CHECKOUT_PATH}" diff --git a/python/fixture/server.zip b/python/fixture/server.zip index 1349ea7eec5ac45565ffab35b6e27b8a5f69f34f..b1aa4b53d78fbd75f3e0aa11305c13a17aa32644 100644 GIT binary patch delta 10634 zcmc&)3ve5Cdf$~~OIEyz9ov$u=UTS&B(fgX!%j>@D5)V2Oo0%ROB`8}ABk-#vYi;X zG;zw^W$rGy!1nm(a=gl93P&e%*-5|s?%c(b9(HA z<>-A2qr>A-KFs@qkw7>U_D6htv^5mvz4Cw2khirpES$MLy0~X7F#f>yU^>c2rT>l5 zB=85?yupA%#-qVMHHh-s!or`GiSR zlaoG=4yT5bppAa;DoHLIYUUO32|uef2`kL=v_LJ~X|~l+BS4;#&tqy<>$@vdCwSKk ziB<=~0b#DuBAi~sFG39=nLQjZM$=$5pp@wG>>5b!2GioP%wV^0=CL|+H|Xv5h1z@~ zfYFu~`Pt3#vlfrX16(aFBcqvQY-m{`nNE$i0H^psLew9U)wcS)!elKPsAH+33R#%& z^r98wy_@8FQf#PUV5=Bf5R$(XIAcAyH;<>VLNtB+GuY@|297zrC!bDJDs(|4{(~>#hAI(}n?-#s{yvEB5XBOAztt=q7)gt1( z1A|G===i8IiVaF2(M1fVC^;am3L)L#E?D8m>g5=P3A=NuX39Nk2NcN%Wp|e3D{cN#x4S4K|8Mjhceh|gRg@Jh66ku~3E`UC&-$Q1t+{?u zk8rJ#TMvu0XsH$J3KZ0YSSH>-6vJ*^f^7-J%Ur(%tBBaai;@`Y%-5Rc*V8&vlH#wb z>r16M?Y*xz@n*3{-az4Iy~14^%rjM0Wuwh?Ihy7|wAoq8rfJLb^(?(6vC?DZ!)iMX zUZw?2w?H@m?CAy;$b$S-HI0-T=~s%jcJGp{Gr?A(8Hs%@Z;jk*LcW01YdRvOdQF6n z-kVOuMw5Iz1*Kznbc9d!mgq7<^*Y`d;eq^mI!q7^noTh6Fki9AFWX@pB??t;O7QW} zrYrBbfk;Gh+12Xx`tvm-==;(wWftqA{C(3diIPbIE`3#|?VS-gxljh6rSiU8BDeV> zf8q_TUKL*5>I-;%ZL3!YR!2hN+uGU!-T?0p=GK&AX%>TLj23jADzWVprq2xR05TFP z*(|AEvc?M7cK$!~K-<|{*b4jr3&u=pJLrVC4G1By*9mK>zM+_yeI6(bSP(P2qQaI& z=O5Jum`6IXbilSOliU@D!2&8yiN2S&9}`S0e8m|<3m-JU04;pj{E8{4&(PRWU4LxG zeTmIpA~bR=ZlKd)K1RNDr~4x;J4g#$Isg&ubcN;xb@$BDAaQpg{Z~3N#{9dC|9Wts z>)eWezAC(SU!Y1)UvDrB3x8jE#QnV&t2h3_CGZ=KWO9ntmGMlcQ!l);yH3;Tghz{< za~A>OYGjg|l2xutc$-RM6no93@?+el!;@0GcZRV7>tb^0oWc6yOOMsFCdsh8-Oq#F zFaC}+Mh*Bjrhfa)SB0vxt*j&a?pgNVU9989WmC_%Tq@~O-sTMs$QCgg^nzj0djp&f zIPu6UMO8hgJ{CyG!iHE&cF2OdEZKV_tj$@o6^W6t4F8c43Fj41`SESNTbWr3Axg^r zdILMgnk;w?N8So_J-cZm>lS@)IdY6A4bC5E^gD%dGufGb7n+0zI)_KU zD|_N3yWZ7O10R;)9d#@i7Kbnf(hZ+8w{-VzH3-!k>&fkB*-rA_X8n8}wdL8fY-5yO zk`oSNP4-=jX1@42`-hF1=G-O!1C2X(x%w@*q=EZ~Rrmggr0ca|wmSPl zo^n|tR3`>ri&x@ChhZZmcE8qXs-{m>sC@r^EuYhI>Id2eQO95QcX8ysx3s}32}=aw zf^G5ILYyp%m@d1L1a)-TU$Lar5AcwZZZiAGIP1fTvdghlx?c(gveTxoW0w%_qR}ko z*zR+>T#jX*w3bPU^xCGfCNaqSIyUSfA0I9Ai>u)M%0E9;XpqCr9Cj41g2T=OuEUNh zCxeEi9BX+nY6G05u#!uM2A)?S5lh!OK;H$Y^^oQDRQA7X*rz_3>ia;@){{HWl{wcb zyA}hj9{CuChx0zXDB--U2T`MXv>1B~F=;!%s<2taYD#n7` zzN!2RYCWw5SHA6HNo;d@po$Y^(!42Kck3sGmcu&jcPdGcy5ISnGyC0#%GWx#V%GWa z9`$aPRhWBcEP-wWHsoJ{{0KW!ysGd`pm~}%f!V~-asz9UY%40+`+r!zP)ydq`ot&X z#Ca(=*~{yGQ{t^=4<1PMv-)pcD74z@WTu=-5+v<%J?EOjap9b^ss%ptcvCSE zL;(z?5w{XQ*U#>)>-1J`1c>aw3l*&NwxLnmy#Bw+$h-QJkl$ z8&Mz}IpvHO3#Q)1cfh^x!PySyci}{f@UJq$LD{Q(gpD&LVJV=F+bhhpd}I943RS%o zsyEtAlAU>fa6%Esl14{GZY&`*8`f^Akg_5EM1_mBbr%?Bnxlrh0g&TYbWf0r zTa1OJ$?|u9eV{Ng%IwTW7ni-V$M_4E!2_0}D_~fIRb3PQ2@E@Sbodg{K4}b-Q%BfJ z-6OiD4+;gE?aV$`83GE21pYqPNx6}>I&AQSelMwpl+lxqFBom)Hy4ailO&}On(U&B z#)aZSAa^_GG>YB!LFb$t9%AD^xphuuZngaK$vF+V%m27IN9xJ7u1ZM?9RdHWyK=rb zDSUM6!4@+3RAq>|nEmK1`?jlEvPJT!U-rjY)dl_p&|O85n6FMD91&bN2GV&Q>G9Bz zVVmA9P41GKB9{+es^mrEPV7JSJUQv&a%0Tz_Kv4j_;;$tn5SKxXzwyt`R$;xoT2@A zgBt#Px}#~5uk)%P-rLnIBSFcB}&jc zFb*$hlq4t{_ratk&6zZw-xIcvGtt3{ZduQ4=S=QiRgOK^yRW4}SBnBJ%cgR!#kG zvbLYA7omF!?$}plDH}LFI~cStS{jL-j0-=upr0RFCR8+OJtN zPQDm`kjeWi44HH1Ny?S`7D~klsN3vQlw>f~*O$zwC@6Z9kuuah80$$6swj32a@O{L z8>bXKsl;etxu;0=4z5;=6wOfoXH*h=pV2$eS1gF` zOE5$E``gOVBnW7K0r04fZUyLi*!uvb!Y}5PqA02pDbCt_>=Y#!O%A08;cJxLDp_>c zrW$|!nWQnUqM|b{)%a%OWy&%zUJM;I^BrUwxI(VN zUy%v#@It!*^@tfWkSMT-^3kL?{UAxh+RKlpm~9RJ`ZDEBq=sSx!%8a=JUYE$?r4mk zlM6$VrF8N6msg}LVdI<2ji@l4Nu`sS(Fc^a%7YccuDSXUX-RQ)26-&P)#_?^m9k6G zRkBliF$FYtTqRGWr1tSD+;NPpn1fok4-dxTJ5)?GijS`#^U@MFbd9|Bb;<+@St1x% z32Yy}N+|{tv4R3aEurOa-Ze@vIuPHHRKY~3v7rv~L0U4FLKNFmiN@x>;w{JnECqCc zqT&4ZX|iRfWHOc%=TENXSbLRDA{U)(kgNS)Iye-c`+b3u4()a?yGv+*kp#~waZTqI z_7{5yo8g>1QoVn)y{j4SmKp~NYece&^+2~CGTgrxeX{z+0k+5AsxpirY1PDHO`redUAe4RqQvp z0vfc5{URG#Ho~nGhmQ0J+&?eg_mRIH;pT|G{UhAhwEYF~jdErfMXD8d_tU#M3@F{- zi*)2L5K;h@meb)jXZo>;+K0)RQSKJL)WnkaZx^L9YjWcX{?1M5Q875uTTB#iY~r^M z6ZaUmK(u(9g{>vq#<+5=^BWwACOI?NH^v!N_^y(aQxRQ2`=3e9cK!Gs8Uq~9;_$Jj8s#{Y=+e|0Z$@~pWkWYUTH&FizmzR9I delta 4194 zcmZ`+dsGzH8K2pmeX#6RKv;2>-C16~np!CWDq7)G(>CNmlH(&%Gz+^t6j{2gyy^>l zVnk8YFB*)GL0C%T5~vUS~Q_2wo$Qa)BD}oVP{yKbH=^g z`}Y{aj>?*iBjZc!}@A3bX=x?cmKO({1nYPiu=L$cXWGMFr zYd9Y5qqZda;uFz`dj2i_We6&A`CL9xK9Gxw66K8xc&D2$gH<^~p{aE8ze>3-{`qA2 z`w+j9cPKPy)nL@})&@SUz>3Sk!IzH0$y0m|EIVVf3*P?2tM#ELiQI9?#C%cM!R2xJ z2oLcln>gyN~P%0#kkEE}ve&me8MX8zaZ(84{Gg>&HTWCBjCFe5x7W%IdSLIdS+Dbpi=fciLquW8)X;hHs z8jYT?jhnJDRk2aHv1bw_3>kCcRBDj%JMSE-(uXZ&aK$by1Z@<%K6UlD3*1wsLrB06 zU2yQIYf?Z)f>u0mlVSSH~E63gAZ_QDdJ31tUnUg=8Zc^#_ zuE~_4xc(gX#uV_JHvL#vtOo~k?&@8KbqjbQNi9hX51&r=7KuZz>)p^hWD@1;_e{_8 za3S9oPF%r#Q@C0Whmvl?@)_o7IzxXx{rgP4p|L3(Cylfdw(^x5);5`kn0}VrmTw29 z!p>%Mc3iCD5W8wqetYt3z0$-CZpf{LAX@I*XMWaA%We13?jAGmP|qW`gVo-Lr_B=- zu6*vP+o1pch!AcMUcEZPnwx8i3PFw+Ef3L}4_3^utP|A`Bi8UPW-aNy0G64SY&bZ> zY!^b;N*3#NQ+kRet0q`n7E8y5?~VY=nk~m1DkP$w1v9b5lA+#~UWi_}G3`^EVO18}F;niDrhV?JX}-ZOd+Q^s?zy0j2PPnBMGzv|-L z@|0`VjFc0_4Jo|5Y{<&lM;Y+an1W zFn!ecdxoo}77Pz<1r8OUk6K>z&^AG-16V&RHeKl`@6L)1hpIH7fmN~gaIp-XiA@bJ zU5{ebzB6~WUDd*xciU=vf~I42|IV)KSbdM|9?kKSh~RtekePCz;^cL(C9dPsHK9dBdwrIF6^n1|r5|;_nEv zkH=Hy@Xdyu8%Qz??I99WP=I3Qs}1C*iuh9-$rNSpPa8>-vRAZ;ct@c-kq-vw-9#p^ zF^S&^aJiE>3?D88eltl5>)ArY)GNSozOpG5)lETvLvayrNHj*iK5+8GrPqkV^S>($ zRud>+UOy7W=6e4&;&5jq^QcUja7|hqEAF$KiRgKD5ySZWb%D@G1e@nmw@^oeuS5J6 zGFEZ)%vR#`e9pw`{lNx*aHJfYNQY3aFB#TuA>$OeZW^F;bCCgkY77LISJzfXJJr`g zL_@+pxJ|_sG2HP?|AL8C1_LYWwC0dwWz?g&CF7v9T{YLzPDF9@E5iLl7^s`gV60;23ndKY^VgS0u`#%V zh?$``7$D?dQCH&)`5U8ju?dS74F|8mc3Qg34>lB;g>y~|`cdU=@JCrd8r-)^7@#f~ z2>Hu(G(`c-CRM zq>H5txRQ1UEeL&QgNwC>+Xvv%4%K%o6N7jIVmgU4A8|Og)F(z=(BHRP=g|*3>cz6! z8gKdXk?Mg}LyZCn1*@NUl~Q&uVQOe;KPIEt8l1gvGDMBftLr!{~-#)q-M$ zZkFumn_B46cR!N4tfkPoQw^Vaz4#y#4*Kf@HIJ!t>^p|GFM;1h5;f_4MX0i)1p?#iT1A zpQWex(ogl_=k3e+sHF;3rMUoi#u_@u$=8XVAOizgd4l* zA!Bop9vTjo!;5>!SRB*wn(YC$JyElhNo=!!twfd&%^j%i5bAVW28FX8s>7#E7K; diff --git a/python/test_perforce.py b/python/test_perforce.py index d1b197e..e1deafa 100644 --- a/python/test_perforce.py +++ b/python/test_perforce.py @@ -90,6 +90,7 @@ def test_fixture(capsys, server): assert repo.info()['serverAddress'] == server # To change the fixture server, uncomment the line below with 'store_server' and put a breakpoint on it + # Change __P4D_TIMEOUT__ to 'None' or an otherwise large amount of time # Run unit tests in the debugger and hit the breakpoint # Log in using details printed to stdout (port/user) via p4v or the command line # Make changes to the p4 server @@ -104,7 +105,8 @@ def test_fixture(capsys, server): depotfile_to_content = {depotfile: repo.perforce.run_print(depotfile)[1] for depotfile in depotfiles} assert depotfile_to_content == { "//depot/file.txt": "Hello World\n", - "//stream-depot/main/file.txt": "Hello Stream World\n" + "//stream-depot/main/file.txt": "Hello Stream World\n", + "//stream-depot/dev/file.txt": "Hello Stream World (dev)\n", } # Check submitted changes @@ -132,6 +134,16 @@ def test_fixture(capsys, server): 'depotFile': ['//depot/file.txt'], 'desc': 'modify //depot/file.txt\n' }, + '7': { + 'action': ['branch'], + 'depotFile': ['//stream-depot/dev/file.txt'], + 'desc': 'Copy files from //stream-depot/main to //stream-depot/dev\n' + }, + '8': { + 'action': ['edit'], + 'depotFile': ['//stream-depot/dev/file.txt'], + 'desc': 'Update contents of //stream-depot/dev/file.txt\n' + } } # Check shelved changes @@ -332,6 +344,21 @@ def test_client_migration(server, tmpdir): synced = repo.sync() # Flushes to match previous client, since p4config is there on disk assert synced == [], "Should not have synced any files in second client" +def test_stream_switching(server, tmpdir): + """Test stream-switching within the same depot""" + repo = P4Repo(root=tmpdir, stream='//stream-depot/main') + synced = repo.sync() + assert len(synced) > 0, "Didn't sync any files" + with open(os.path.join(tmpdir, "file.txt")) as content: + assert content.read() == "Hello Stream World\n", "Unexpected content in workspace file" + + # Re-use the same checkout directory, but switch streams + repo = P4Repo(root=tmpdir, stream='//stream-depot/dev') + repo.sync() + assert len(synced) > 0, "Didn't sync any files" + with open(os.path.join(tmpdir, "file.txt")) as content: + assert content.read() == "Hello Stream World (dev)\n", "Unexpected content in workspace file" + # def test_live_server(): # """Reproduce production issues quickly by writing tests which run against a real server"""