From 77ad42f1ce98c2e59146b837ff1653f30b813459 Mon Sep 17 00:00:00 2001 From: Olof hagsand Date: Tue, 23 Apr 2019 11:57:18 +0200 Subject: [PATCH] startup measurements: test, plot and report --- doc/scaling/clixon-startup.png | Bin 0 -> 6209 bytes doc/scaling/large-lists.md | 35 +++++-- test/plot_perf.sh | 168 +++++++++++++++++++++++---------- test/test_perf.sh | 10 +- 4 files changed, 149 insertions(+), 64 deletions(-) create mode 100644 doc/scaling/clixon-startup.png diff --git a/doc/scaling/clixon-startup.png b/doc/scaling/clixon-startup.png new file mode 100644 index 0000000000000000000000000000000000000000..5115bb67445df60f78ff69a4a44c4de562976145 GIT binary patch literal 6209 zcmeAS@N?(olHy`uVBq!ia0y~yU}|7sV0^&A%)r3F#IJLWfq~I9z$e7@|Ns9C3=9hv zEC5jl7#JG;@b29K2Zj?T7#uSn{y(6=(BN>O;lDvbLpKA1Lj%MAhX3{duPoKl z`v0H7;lTw41_mi9DPv<}Utiy}w6yyA`k6ClZr;54z<~p2&YXGr^y&Zq|34TRO?Wq9 z?Tu$2LO(qC|Nq_HrvKXxu7CIL|8)k2{|pQ%3=IG4mAaMmy2HWzZb``k28IR(hO_PG zUNeB4_@Ck2#s4Cc|JMh6XlVHF%J8e9=;gGk56^n`ZqMG)#{Ko{odpdF|CchnYkD_@ z;qu)DcP9imgl6kEh%hvWh%`8eG&nRgBphf+_}>6>zr+6p4*wS<{9jP;|GK1_sUokH}&M2EHR8%s5q>Pnv;2Y?G&pV@SoEw{y9hRx1d(ZnfUaW&WJ~{=exT zSnLCnMFhGQ?MYu&n|XW5qY5@I7Y$GhG2jIoHI~aWXsrCbDaY!mR($?e?gs8%`*cI1 zP1i1N=)J1PaBih~_NuIPO}>`Ps{StDw6guW@9imjXWd`W##H=ct(< z8HvsI&-(rGEyJ7(*H54JJP;;*b!&6I^F@n4BDc1$ir>HR-SNf0qj-h*j!)mkRj)qr z+0lSa+YAI^F6g-4e&io6{#D_s!i9vzVYw9xPkjx1UC-`$-1!^Rwad2|Ef(I{XR_aH zS)AV1Sy6k6ies){&H8=p=Y+Wp&)#})AGBmwO$eMCx^YsLc0ypUwHFD>swRgEz z9J5W1>W~qLSy;``y;MG6dsFF3)2&Uu#%0epm;SfvRb4>EpVjxShVNRuY)14! z-M^fTXKvi#)O7Fl$$xN%e~0J>+24%pJEi_sRiuad)nDDs_FHPP!7oO~pbLK(JR2Jn zGzA2($A^Q^s!W!u@T?CO;tQ6p4q5m`*!+EF0KZ3}h2`skz=hkhBN{lm{z!JZEy5f0!oKr;k+WTNq-XW&>aTLiNtT!1J!EQXefN5mk2Rxt z_9UIuMIYBnuYP@Xp0|R+j32wSF8duws@f{EVNFnymqYAS`E}8sULQL6y}9Ah&rMpF zLpg7pS*@;T`C!xYS?$vs?g$&d3TR(#)I0le=uEfR?n0ltN)I+Ym*VIMP`)3YYV`Ex z^w61KoeOQ)9?ai- zCW(Quk@X$(hujs`92|Zhr5pBJ7Tqy>yW!>k-sF^1g&YaDHK5|Qn=WvL_ul)mSMGn% z*Z2I}cXh45`b}se(rJwyjh3iKQL|n#dO5N8V3LlodJ|0@Lv*1Io;sf3}lVp`_z`A6j zN?pTe?|tliP=m4Z;iNYc9-1=zuYKZnKv8^-$y~QZO6*%2^$e^3Uig__7*@ypVY_=C z_vvNxmpz&hX6K-NgYD2X#zce^>zY%Zmy168Texs}v^#G)%;|QWiRqr>_SZ!w6E1FT zXfT)fEouZ}3TP^X^8I*pO>nNFzy~&g4|YndDoTfUO_vwAAkQJe!103j4|k&nqvxij z;;}0ZsZM!b@MrSM57(<-U6^k8;(6$2CePAaQzyykCM0isJ{c?%ayarp>ROB1*&l0H z8#<}5ZCKr~x9#!w$JYMK55>QK8GVkaPxVKEP;9KoXNyf56E7V;XTSg9kG1TfPyBj5e|_vWwqTf{DV%G7RAVZ8rT zUY@bG{qgkVuJ5&4e^UErDp%D@{*STvajivl!87;&>u3CbHdX7#(f>c*9sU@aEce}Z zHdp;kW8eOO*t-7LLIpO9Gq<&Fuu)`xGM(Yvly!$cwg;Vk(x4vvF#mhOImYb0kx>VK zr-~{j#O!H%EWN%&`$vk_pSN=7);?G&A274-z};DXMMVd;*4Gp{tbh1paYt>TmFT}T z{^xZE7Df9d)d^}}vMgMCV@iDu%a+ziUzJaB1u+p}2fi8X<`cLu^~~SRd-AT4Q|u)%(r36cUAdTSXqf*lS8D6#e&ch*YaI5D-I!-IpRgz+D%`;vx> zSrhyjQ`OR192~5Vd0NaBLUyJo=Ko0wK?gKUPZV-$7#k|H zFhzYz)$^Lkq{N=$zO6X0ijnc=N0F~58XL+S4tR8k7}W57x@teOQ&LgjLfiYN^L-d6 zGJkqh_*I|f$R{?f(~pjJ+ZRhNZ8m(;ejf@h=8Sn^1KK}X9qi*i}qd$ z4_E~Z^pse6BAR1`gTgJJfY1I91yqF9bd*U zMN^^3fzM_KQ(FU1#fJJR4hQlb9E4oBBo?u4{NkUM_1<>rAVbyuY)?F(!>^ z%FG#pLNVQ|E@d066HDn4dud&+Kh^oCdhh|CVu#+ijoV(dr);<@J;PVDXJ>%1Six4W z(%t#2*+IIc?>ANMtz%=JT;I!{)1=?F;>@de#=HB^EoG^>=F@PJbJfv3>$YV!Z>c@v z@%+we$2oTptv0B?V`7djj`K;kG*YT_sU9*?0ePF%do$e;l3EoSh z9UP(*9*F&L@9|*FzoM{?>6FhIj>ZP*IouAQ=<;FBN?>O3VfmxX&a|$f{!53*h3#w~ z1p>e!Qu8>N;gpDgWr9!u^Ld8@A_w-1I2@4tF`Z>1Q`J?4I;c^?^H>*vBJ@(TR0S)G z3j04*CZ;}z11=pR7xuD!{00f!D?g^!8?n0ZdGKgG&=r^<_G3C9M})(HDIFpgHnQz} zJpas*9s!Pu^?xRV;>jz3v7UjEv$5W@a!bPuJ}X-#)-CHC-}$ra8XGDRP_-%LzQ z)lAuerSJLgaP-^kFskKi|GdGH!M2aYb;l-UM_y>3$xaIkHCidnvEhCUfE8b=k?uX`tQ07 z4vsz!&4dmQ#~arb1X)1|RjJ6}(7T|cIt-~?C-m550`~sXtS+s6an|FYBkvn7cgC*@ z%?qbSf4!*O%J#4DSXB2zho!g0@5wnj6f|@@rdczplCMe}9i#UejPj1dU9S_FFhBt+}jJKctx-fep6BFaj z$1|9>`O4Z}s(mKaasXT$@vVKZPe=OyK?4DSg?$2d`8%X;2=_6q`gg9@V!Fq{hI+|_ z`y4Cx>|P+N%*xD^)(~vYFWZLbB>@ZA zzO^lF@J;I#_%WTeNLo?AfcuG=Qw9q-*tZEfg)p2FnUKmiBT|{g^aHO2*Nr<>N+(r= zBoq?lJobw7{5Nu3(qLZ#_Gx3ocIQAv<0)UyaD1QVwY^V-B>;Of<@-Z?@C4XGp!4lR^{2OGd`KL>L^!yb3=;j0w2boPaDG8ulm}`m0Q%a z$BO=0^x0oyCfALXO|1 z{zQ9)6YU40dgRk@Ddg6FS^na{{yiN^38!QNxHXKUt4frQyxaV&PU@=Z%BX|O7^Y;l zo(VX_ro4cOkBQ|AE7P+!9ma?8WgS8V-~7|IwYYKo_7E`GA25lsBr8WoryoON!nG9L2R9fR8!abqZ;1bP8f4}}v6tBw z)h~0+TLS`*Y97s-{D=0$cDUEA0OXZW@=xkRQQRX>7MG6hPkus z)~#;$z3bXlqN|X|>!hI|rFc#L`irkoSL+VGa!}Oaa#-d4)!Ipfw^!hGe{{aH`=e0C z$v54(S^jXT|KCz;#H!K~;BdD8-~VfK&%S=g&T{+DzgOY=SMGN^H#@WP<%A>W7~gU^ zG)bkbs|K|aTm(0)Zub>kD4G+==)OPX{T2UZ4LhGVochAN>chl}lm@40Ic+1+4LZ-g z_U4|dwJ+)L(D^fYy}ahkJKWC<1ZNBK|6)}WX1uGG)E(UXNA4x4W>*nsoV!f1OXWe} zOK|0RV8=wE>-(3K-wk4BwD+_Ty#3%_@SDY|;*6L5j-++jaY!`#u$0X1&@JFKy1J(w zoN-RNIxJt@9JlK85r0rk@m*zgt^DP8Hy9Ypmwb9Z^Vh^ltqq-jn(J6PL_Xy4J}S8E zo0jC}!N+7J-oVJRMfWeLz9`_WW&JZJS}wk!A-jXM=9=&JCm+No>|toBow7mv!twpq zrX7zL=RsO4)9$AV+fLCoaxN5~u#L&>sWlU0ddFYoUN)0GtSVpmkH&MkaJ>j_u6PCV zUdz^oeXI+PFZ~3u=Ym5x;(*G>4D3rh7 zebVO)N9C3V#$&Q4KrNm?rjGX7{>BE8e-9Jm`E0@tY;Lb*l>NiPV<3FHlvilYQ*C2I z<(BykdBz)Mi%M5AmF+np^5LOn!jsC53-8N0r#95Pb=%KM;9GV43;(aeNBI|;PV9f8 zs*rZ$>4|vHr}LfpDyr4C9hOo#=W9N*Q&O;=`RNB|;j_C$-DySb8QgJW7mLU&OKFTi(s%2JAfw3kvzaEq!=m_tX4|Q3vNnZa9{|cE^K%e;>_r z=Q_71T6n4ahH!T4w@R!Z^9yrh-yRbT@JSNmh<-0NgHvMJpV?hDU32`GHN0CGvp2d! z=|SCGr{5k6SzNjwi(O-~60eXyBDmr0>1E$KL=M@TR6H)ezq{2Uv|wphS^fgoB@R#J z-|Rkm|IhvN`&&NeDc(zHDZ9^gZO;0K|F|{?{dxcX{CWMmyFS}zitp{;^i%rH@94MD zd$?sY;_jLYJ$<>JokfG8ex>~8l)FYBQg?X@F5L55rK@2X@1<{j6P2t=Z{a4iG zx_s8-4jm1*FX_i?KWg9P^-=DJPsjRtiRU?;!IvLZGF45lpAqwd>6Y1A>zmvhfBPQl z-Q(eBwEg?L?6&#tBRUByVvfvCR9sNb^dohSv9TWm*!@R)Uh(_K(f2*( zF!3|RecsaeZTG#0cd~A+HxRgRd_TLH?KWkWp!(SfOT;f6-mm?4hf3N*1vy0rk2a>b z>2*acJ|<2N6fGBA_h~r4qDHu1prWH8&zRBf(K6}Df=n$-);7gHJ2OltdHe$06vbin&Sin;BBSS<@qkNrJA|1yEwpwf)a=Qb9Wu4D?^ zV#Hb#$0u>Y?w04h)K4;u$#*{AC_i`5`kaA)!4KR1x4cmm$7elZI(DJ|g7wT!NkvW8 zo~v{`V=l1a(w*|&>Oct^sV@sKEam~EmajOKD}66Y_HjJN?nA- zWyO+(eXJ}G8YGzHo;5H8T)cXC*(<$1CIul-X?i7|*=E`{&F_CffEwwLV- zjLo;qE^V1Ces4wIuFO2)r*S(hMGnpk{rtD-N4jad(ylmp8NN&Wj`sIIL@UkN^I=)u zt4mg=_7-@r_>q1sbX7pDZ(*3X#;n(+A6VON$Txp+-Z1Ba(8DLK=5uGUO~2`~Q#_AT z^#7SJv(GM2oz2ncbTMYV|03QCQEfIlf_`y#7H;V)vc97=CD*4u$6qZ)!}rbWt&a;j zvN&dKlmFbRdgx@g^_1JWX9^}an52h=x}=-7DeaE4Uc`BW|D@~c^B!|R!rxghM)oc| zvF>_b29wk+w*?JtH`2}eq|_`|Uu8(`dc9TolfvJi1il@6`oz6Tr~aJ&V9m1tWtQvN z?&0EvY%gce+4eRZO)sO^jV{yOHhX>b!TR@u1Lo`j=&)@093=(*i!8&*577;+&UY ztUe;hQCSV@0-2de=|OT<~R2py{e~foW44np0NIDSvtYjP~2PGugOw zg#;SQrBlAFi`XksKYKaDi(OKp1{}USYfBX#T=G~a^`t?;WIEHj+ecQp@^Kt0(oDE7 zC(q}=!N-zw>udfl)-@%sCDtYgG=85uVQSIYT@q_%U;1)#%L{fE6dweAkk{w^=V|+u Q?HEYh)78&qol`;+02oshN&o-= literal 0 HcmV?d00001 diff --git a/doc/scaling/large-lists.md b/doc/scaling/large-lists.md index 35424dc0..7bb3653b 100644 --- a/doc/scaling/large-lists.md +++ b/doc/scaling/large-lists.md @@ -12,10 +12,17 @@ Olof Hagsand, 2019-04-17 ## 1. Background -Clixon can handle large configurations. Here, large number of elements -in a "flat" list is presented. There are other scaling usecases, -such as large configuratin "depth", large number of requesting -clients, etc. +Clixon can handle large configurations. Here, measurements using a +large number of elements in a simple "flat" list is analysed. This +includes starting up with alarge existing database; initializing an +empty database with a large number of entries, accessing single +entries with a large database, etc. + +In short, the results show a linear dependency on the number of +entries. This is OK for startup scenarions, but single-enrty (transactional) operations need improvement. + +There are other scaling usecases, such as large configuratin "depth", +large number of requesting clients, etc. Thanks to [Netgate](www.netgate.com) for supporting this work. @@ -41,7 +48,7 @@ The basic case is a large list, according to the following Yang specification: ``` where `a` is a unique key and `b` is a payload, useful in replace operations. -XML lists with `N` elements are generated based on +With this XML lists with `N` elements are generated based on this configuration, eg for `N=10`: ``` 00 @@ -66,6 +73,7 @@ Requests are either made over the _whole_ dataset, or for one specific element. Operations of single elements (transactions) are made in a burst of random elements, typically 100. + ## 3. Tests All details of the setup are in the [test script](../../test/plot_perf.sh). @@ -76,6 +84,7 @@ All tests measure the "real" time of a command on a lightly loaded machine using the Linux command `time(1)`. The following tests were made (for each architecture and protocol): +* Write `N` entries into the startup configuration. The clixon_backend was started with options `-1s startup`. * Write `N` entries in one single operation. (With an empty datastore) * Read `N` entries in one single operation. (With a datastore of `N` entries) * Commit `N` entries (With a candidate of `N` entries and empty running) @@ -83,7 +92,7 @@ The following tests were made (for each architecture and protocol): * Write/Replace 1 entry (In a datastore of `N` entries) * Delete 1 entry (In a datastore of `N` entries) -The tests are made using Netconf and Restconf, except commit which is made only for Netconf. +The tests are made using Netconf and Restconf, except commit which is made only for Netconf and startup where protocol is irrelevant. ### Architecture and OS @@ -118,9 +127,12 @@ The tests were made on the following hardware, all running Ubuntu Linux: ## 4. Results -### Access of the whole datastore - This section shows the results of the measurements as defined in [Tests](#tests). +### Startup + +![Startup](clixon-startup.png "Startup") + +### Access of the whole datastore ![Get config](clixon-get-0.png "Get config") @@ -196,11 +208,14 @@ system degrades with the size of the lists. Examining the profiling of the most demanding Restconf PUT case, most cycles are spent on handling writing and copying the existing datastore. -Concluding, the +Note that the experiments here contains _very_ simple +data-structures. A more realistic complex example will require more +CPU effort. Ad-hoc measurement of a more complex datastructure, +generated four times the duration of the simple yang model in this work. ## 6. Future work -* Improve access of single list elements to sub-linear performance. +* Improve access of individual elements to sub-linear performance. * CLI access on large lists (not included in this study) ## 7. References diff --git a/test/plot_perf.sh b/test/plot_perf.sh index e4a95acf..9d394746 100755 --- a/test/plot_perf.sh +++ b/test/plot_perf.sh @@ -270,78 +270,142 @@ plot(){ echo # newline } +# Run an operation, iterate from to in increment of +# Each operation do times +# args: +# =0 means all in one go +startup(){ + from=$1 + step=$2 + to=$3 + mode=startup + + if [ $# -ne 3 ]; then + exit "plot should be called with 3 arguments, got $#" + fi + + # gnuplot file + gfile=$resdir/startup-$arch + new "Create file $gfile" + echo -n "" > $gfile + + # Startup db: load with n entries + dbfile=$dir/${mode}_db + sudo touch $dbfile + sudo chmod 666 $dbfile + for (( n=$from; n<=$to; n=$n+$step )); do + new "startup-$arch $n" + new "Generate $n entries to $dbfile" + echo -n "" > $dbfile + for (( i=0; i<$n; i++ )); do + echo -n "$i$i" >> $dbfile + done + echo "" >> $dbfile + + new "Startup backend once -s $mode -f $cfg -y $fyang" + echo -n "$n " >> $gfile + { time -p sudo $clixon_backend -F1 -D $DBG -s $mode -f $cfg -y $fyang 2> /dev/null; } 2>&1 | awk '/real/ {print $2}' | tr , . >> $gfile + + done + echo # newline +} + if $run; then -new "test params: -f $cfg -y $fyang" -if [ $BE -ne 0 ]; then - new "kill old backend" - sudo clixon_backend -zf $cfg -y $fyang - if [ $? -ne 0 ]; then - err + + # Startup test before regular backend/restconf start since we only start + # backend a single time + startup $step $step $to + + new "test params: -f $cfg -y $fyang" + if [ $BE -ne 0 ]; then + new "kill old backend" + sudo clixon_backend -zf $cfg -y $fyang + if [ $? -ne 0 ]; then + err + fi + new "start backend -s init -f $cfg -y $fyang" + start_backend -s init -f $cfg -y $fyang fi - new "start backend -s init -f $cfg -y $fyang" - start_backend -s init -f $cfg -y $fyang -fi -new "kill old restconf daemon" -sudo pkill -u www-data -f "/www-data/clixon_restconf" + new "kill old restconf daemon" + sudo pkill -u www-data -f "/www-data/clixon_restconf" -new "start restconf daemon" -start_restconf -f $cfg -y $fyang + new "start restconf daemon" + start_restconf -f $cfg -y $fyang -new "waiting" -sleep $RCWAIT + new "waiting" + sleep $RCWAIT -to=$to0 -step=$step0 -reqs=$reqs0 + to=$to0 + step=$step0 + reqs=$reqs0 -# Put all tests -for proto in netconf restconf; do - new "$proto put all entries to candidate (restconf:running)" - plot put $proto $step $step $to 0 0 0 # all candidate 0 running 0 -done -# Get all tests -for proto in netconf restconf; do - new "$proto get all entries from running" - plot get $proto $step $step $to 0 n n # start w full datastore -done + # Put all tests + for proto in netconf restconf; do + new "$proto put all entries to candidate (restconf:running)" + plot put $proto $step $step $to 0 0 0 # all candidate 0 running 0 + done -# Netconf commit all -new "Netconf commit all entries from candidate to running" -plot commit netconf $step $step $to 0 n 0 # candidate full running empty + # Get all tests + for proto in netconf restconf; do + new "$proto get all entries from running" + plot get $proto $step $step $to 0 n n # start w full datastore + done -# Transactions get/put/delete -reqs=$reqs0 -for proto in netconf restconf; do - new "$proto get $reqs from full database" - plot get $proto $step $step $to $reqs n n + # Netconf commit all + new "Netconf commit all entries from candidate to running" + plot commit netconf $step $step $to 0 n 0 # candidate full running empty - new "$proto put $reqs to full database(replace / alter values)" - plot put $proto $step $step $to $reqs n n + # Transactions get/put/delete + reqs=$reqs0 + for proto in netconf restconf; do + new "$proto get $reqs from full database" + plot get $proto $step $step $to $reqs n n - new "$proto delete $reqs from full database(replace / alter values)" - plot delete $proto $step $step $to $reqs n n -done + new "$proto put $reqs to full database(replace / alter values)" + plot put $proto $step $step $to $reqs n n -new "Kill restconf daemon" -stop_restconf + new "$proto delete $reqs from full database(replace / alter values)" + plot delete $proto $step $step $to $reqs n n + done -if [ $BE -ne 0 ]; then - new "Kill backend" - # Check if premature kill - pid=`pgrep -u root -f clixon_backend` - if [ -z "$pid" ]; then - err "backend already dead" + new "Kill restconf daemon" + stop_restconf + + if [ $BE -ne 0 ]; then + new "Kill backend" + # Check if premature kill + pid=`pgrep -u root -f clixon_backend` + if [ -z "$pid" ]; then + err "backend already dead" + fi + # kill backend + stop_backend -f $cfg fi - # kill backend - stop_backend -f $cfg -fi fi # if run if $plot; then +# 0. Startup +gplot="" +for a in $archs; do + gplot="$gplot \"$resdir/startup-$a\" title \"startup-$a\"," +done + +gnuplot -persist < $cfg EOF +if [ $BE -ne 0 ]; then + new "kill old backend" + sudo clixon_backend -zf $cfg -y $fyang + if [ $? -ne 0 ]; then + err + fi +fi # Try startup mode w startup for mode in startup running; do file=$dir/${mode}_db @@ -113,7 +120,6 @@ new "netconf write large config" expecteof_file "/usr/bin/time -f %e $clixon_netconf -qf $cfg -y $fyang" "$fconfig" "^]]>]]>$" # Here, there are $perfnr entries in candidate - new "netconf write large config again" expecteof_file "/usr/bin/time -f %e $clixon_netconf -qf $cfg -y $fyang" "$fconfig" "^]]>]]>$"