From e95f679d7d9a115f643bd2288c30d1e37cc78276 Mon Sep 17 00:00:00 2001 From: numerology Date: Mon, 21 Oct 2019 15:17:59 -0700 Subject: [PATCH 1/3] title change --- backend/src/apiserver/config/sample_config.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/apiserver/config/sample_config.json b/backend/src/apiserver/config/sample_config.json index 0d25e697841..bfd64662901 100644 --- a/backend/src/apiserver/config/sample_config.json +++ b/backend/src/apiserver/config/sample_config.json @@ -25,7 +25,7 @@ "file": "/samples/core/exit_handler/exit_handler.py.tar.gz" }, { - "name": "[Sample] ML - TFX - Taxi Tip Prediction Model Trainer", + "name": "[Sample] Unified DSL - Taxi Tip Prediction Model Trainer", "description": "[GCP Permission requirements](https://github.com/kubeflow/pipelines/blob/master/samples/contrib/parameterized_tfx_oss#permission). Example pipeline that does classification with model analysis based on a public tax cab BigQuery dataset. [source code](https://github.com/kubeflow/pipelines/tree/master/samples/contrib/parameterized_tfx_oss)", "file": "/samples/contrib/parameterized_tfx_oss/parameterized_tfx_oss.tar.gz" } From 15923c38c9345a659b5d7138eac12ac1d1affe7f Mon Sep 17 00:00:00 2001 From: numerology Date: Wed, 23 Oct 2019 17:29:32 -0700 Subject: [PATCH 2/3] Update --- .../contrib/parameterized_tfx_oss/README.md | 12 +++++++----- .../parameterized_tfx_oss.py | 3 ++- .../parameterized_tfx_oss.tar.gz | Bin 3775 -> 3855 bytes 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/samples/contrib/parameterized_tfx_oss/README.md b/samples/contrib/parameterized_tfx_oss/README.md index 21c38f5b1fb..f53718382d6 100644 --- a/samples/contrib/parameterized_tfx_oss/README.md +++ b/samples/contrib/parameterized_tfx_oss/README.md @@ -9,11 +9,13 @@ This sample demonstrates how to author a ML pipeline in TFX and run it on a KFP Please refer to inline comments for the purpose of each step. In order to successfully compile this sample, you'll need to have a TFX installation at HEAD. -First, you can clone their repo and -then point the version tag in `tfx/version.py` to TFX's latest nightly build image of version -`0.15.0dev` (e.g., `0.15.0dev20191007`, list of image available can be found [here](https://hub.docker.com/r/tensorflow/tfx/tags)). -Finally, run `python setup.py install` from `tfx/tfx`. After that, running -`chicago_taxi_pipeline_simple.py` compiles the TFX pipeline into KFP pipeline package. +First, you can clone their repo and run `python setup.py install` from `tfx/tfx`. +The image used in the pipeline is specified as `tfx_image` in the +`KubeflowDagRunnerConfig`. Currently we're using our own patched version of TFX image containing visualization support. +List of officially released nightly build image available can be found [here](https://hub.docker.com/r/tensorflow/tfx/tags)). + +After that, running +`python3 chicago_taxi_pipeline_simple.py` compiles the TFX pipeline into KFP pipeline package. This pipeline requires google storage permission to run. diff --git a/samples/contrib/parameterized_tfx_oss/parameterized_tfx_oss.py b/samples/contrib/parameterized_tfx_oss/parameterized_tfx_oss.py index 5942a0b0abe..dcca8be540f 100644 --- a/samples/contrib/parameterized_tfx_oss/parameterized_tfx_oss.py +++ b/samples/contrib/parameterized_tfx_oss/parameterized_tfx_oss.py @@ -141,7 +141,8 @@ def _get_kubeflow_metadata_config() -> kubeflow_pb2.KubeflowMetadataConfig: enable_cache=enable_cache) config = kubeflow_dag_runner.KubeflowDagRunnerConfig( - kubeflow_metadata_config=_get_kubeflow_metadata_config()) + kubeflow_metadata_config=_get_kubeflow_metadata_config(), + tfx_image='gcr.io/ml-pipeline/patched-tfx:0.1.32') kfp_runner = kubeflow_dag_runner.KubeflowDagRunner(config=config) # Make sure kfp_runner recognizes those parameters. diff --git a/samples/contrib/parameterized_tfx_oss/parameterized_tfx_oss.tar.gz b/samples/contrib/parameterized_tfx_oss/parameterized_tfx_oss.tar.gz index 02c52d443e433ec267fb9782ead4a5370ca4c324..9d639a25584fd10713629c951ed147802cf8b1e2 100644 GIT binary patch literal 3855 zcmV+q5Ag6GiwFo+@UUG1|8QY)VQpn}WpZhHWn^D;W_Vw3b8{|qVR8WNUEOcnI1=x3 z{|dn`1rGjbdb@i--qIxP2AgeiNw&R19SDp>+uF*KR+Q7`3jgoN4~ZgW>dSU&**kh@ zV~H9LhciR+H{>YtBJ6ts9zLPD|KfSB*Yx`3r=R%$^wR%7`sK~*f5~U!{?XAdKfiuq zy?*fnt_4X@Y*|}rq}sSWxlrVN#4+*0;MhX(G>qc#&!HEN9*>Y8&Ct=yFJ9mtTc5)C z%ft`AzMSI(xhO%$FD(lNL71R~zDw9o%ZgHEBEv6>2h7S3!+1K%Zrdcn&aw6Cd(Xwh ziM@!u+B>#-r<6hxFPwx}$oegPl~c@V4#&Yb5VuZaj1o*_NsGRpAX+BAiikA{V+$i^ zX5GE}kL85(C=76rSU7OPMUdbayD4Qau#y?hi2C8wbC56Vvs^s!0*|pc%*yooR?iEf zMM5aopII{|XyGTl;!%DY~sx$ACH(68NmR} zF%ts&J&40F=`H)#hP3CQyYMj?@q3HJ^T{xJ$}7uAieiI_=i@@9X?0KGA|5;}oG&;j z)}gX4SrY8u_p=IQ0Tk1}sPMi2Vb^AqLPC1WSG3#|(|)7f`GFQVwzPr+T5v!M4qkG_ zVHy`t7jxR1nZ9ME)87Lt;8dh2Q$8M%zo56<_KrqVk=6QCjs2okqlwZ`QALl?(o{6@ zQxtkZQX)!PhT!{{@>ewU)s0lHy4ptG`D4^Gabr1{qw!rr^t1=??rxD`R|$zNJnq`Xc0Iw9NUgh zxwRvd%w##LfpkQoPw8;O2d-gOdgui<9s3+iUs-A+9P-@)T z9{$8mVkb{gO7>4eMfxDTx9!nU7X^;6S^6|s_&&Yw#@=HlJ~>ucR)POBUaA%2Zu&|s zm!fJqBK7b$>@1Qn=2HizbUG8~>Ely&>y{PKh*$9SfCwJy)cYm;K*y9ol=_Eo zRoq!w);;rBib%pmQqIXKzpFVz$zjHs!W(-&LuB@r58qld-t^T%c@wCc@`8-P%pEr+ zdQ?IbQ;)6C6o1@Hf4x*^xtbCwnsP0=@#)NSl20*;qB36~G$;sN%(Q171}&9_>l{jK z>R74r3YIu`tOB7X%$7OVHlI_=65>Ir2Roh71q}A9*ml$+%~?iNzbDAamI(#Fz*G>!<=rhixqSY%!`LI*8 z|8mNiz@;9qShZ%&N?}FjZLPY_noU+;XVrDqyh4*V7SwOc>c=K1pnZ+B&oHLw($Jw2 z%oWIX5M8>ax7pZ0l`#i%qNijQ2J$C^nFb_w*#9GK@jS|amyv{r>>Pp0kS&HvJ?7K} z%=PK&=C^m3SD);QGubMRBfN{ldEQXVnt0fEZ}3FDm)>P9Jhn2-hpyxwuund}`kF%3 zWT@iV+1*$71y0^yUA@0Nw@13Qb>@z(5y4K3lhM?P z2J{b?HkU5>$!I~>{sVTCpdmBULM5k{SAU${p8aN@{BifILf?LOLF>Nycz$zpady68 zW!@Z38BFNJticK*IijOD?KTbq9={$Q4gdKjLl#4SF~`3%AW%W`=IpnSZV3yLQ_;M_ z(}BX^OM$erxlC3;5w$JHZOcJXai1=iQ(s0G%lv*iZOT!~)VmgIn){OG!W(2dB%2k9 z#+ys!G8f`2fX{WpUp6@mJDJ6p%tGHKx#mqU zN)|D;iSIdd5}kE<4r@|ws_|8a4SK;l+Gs&S!#XXFS$dJpc2y+yEL_An(s*CQSyjNp zj5zKX?_d#f?6ZRW4;z1r=*U7e#!Yz0COAkto}1wyEg=4O5Mxfb{erZ5AzHrPYquaR zUj}cAhg9LAYp+Q4m8by>U3*1p#4a~nk!m6szkW3me|WeEC~#AkAGf7w|dn{6_XsEWgJN`d*={6*0@C zWdy2ziqRwnTw*eYnOQocPj`?inwihX(wkj0@$;N}Fm`3=l56v=n){;?Vy!p!M{}h4 zfJ|MH2ZRS8Jg_g}fwC}w1a}|=03iSf0YC_#HUuEngmkvxxv0$^-N)<=hzK0Yw(Go0 zw;CBJXxQ#|T9@zY?aGRdO>aGtcnhYqpmIJZUtAcj?3u?k<{Xv&DU{) zV53ExQnYNZuyI51x+}PB<(3oO&I+y!>Qp4r&&;G-0jJyCkh>arDmRMfk2~e!d8W8i zUO}4>K;f+e-kGcFCphigX*bu)wE|M0menCDqkKwG_^zD9LP0pEzxl6f5WbT~L{238 zyFP{0exNFM9-9qRhUM{6gQ?e?%oXW+VPH7-RO=54Zy(|muT)L|)&p3-_h7wHZV9^p zW)GM>VD^C7*J1WNa@`!U7rVT6jn_BC>Wj<*V&6nc+r;a2?E6t@1d%UFLkcP2%Y;~!R!CE;S2Z~*Ul|CRlC;+oEkWeVlNs^Wr+L`T&({*i zmbrIvgag{`j)#@%1|{toSc#b47Fw^L;m})NeAeYwi!G{pi}Q`G>ivpvSfTmmB8SuW z28X6gz*wNm?JsFjnE;{%nPF4IwFjM6`wA6bHHo}|#b^7c)M*=y?N{1qc)nI!S3g5i z-%{nBA2Ob`@r~3V0x}8Xc|o8>L&-&{Kl$4UvCOru3ReoUAYe_D%Ck+gG_FduZYEpy zi5l2H0sALl|HO9pPh9h+Es+-0e4mGBz%cYQQ4npq`dWX>aJ|>&vvnkgu z44dq3;k29bpkpWA`0UN?c7Dl%Sl2dtEt=inz+1XDs<8T;V*N2UE@MWrfz20ug?y{O zNg%$a`RfGYO;P-&UpZTm2L$0E2*2+^cqK9eht~jn1@INX*9!18&xcfp+lesF4!`RS z80^+CSfwwwiNi`|>Hrk$1rw@L@}n{{#Amk?ItJl`NH}N8uC?oYbCAVoOIeK8JKF|n zj3B!fWY>c1T0bDWmdtm$Pv5tIbf}OHbvNly0a*lOasQD;eIqA;ivTVHxCr246}YIa zENVe1SsFmbg8&)z{@g0is45oZ#A@(O!aNO@;8d*D^i`Ykpqs!Vq$`4KS&%IYvSt09 zvt{kWrbIn_wMPgc1PUO41pzDwV1Zh07Qh0u8_@2(N4r-NV*ug}h&LeKfOuCR-q|s| zI&3se71QmHKV(e0O7TjVbQ5^HYP0Bi@OCwQ_i&at> z1dI7?f+ND*$Ir|)h{MHCTO>6$scWkgKMzGMN|e9e+n%9VQX?Es~1PjTL595g$~+34H4O;A$9 zv1}bJZNC;Z?#SBkET?kLyYaTG#u$BPzL(il4WB>zMk94JV#v$G0dp<+!nvoc$i@V87(_}1ZZZ48wBa6{X zAL2-2l1InGzcLxk9z}k;oqlE)Q8maowTp~X5IKX$*}g>1+Naf4Go(W_4WelfO@nA! zbu=wqTr>#Ah>~z@WZ#dCBk+ literal 3775 zcmV;w4nXlAiwFqOoS4Z%Y4t8vq{wMT(>(Ta?l~ zBo+xa8jWr|zHT%FC!nt5(f%Wuxi6mPI)dxX&p(U*;nM#>I4A-1RBs7ezG*WHco?IkwKGKjmzBe>TIPrtf|FiG-gNI|{22*nU@~h+7L*tVl zejU61UoU4gA~uQ0@TFl8&+{V^!MlwAG>jltCf5Htzo)!>-w!8)(gBwuC+LwG z?uF6#%6S!D@HS*$m9T$-yiuI|yFhrRo`^_52X zI#kwEmL&UMx=96T0TRMrAbjV4)HXRKpRvy36_lG$=r`!j4^UueXa##vum=TuFNNZ8 zjSDC98T4kNZ%OI+chB$yl~j~59}mQx!|kTMqtR4kwLaCte$lGYL|IVLiXMT|Ae!(o z@EtEI5haviG<il0?m>4zsfYqeZe?7uJ+PzL!V=wqo?yw}J)@Vl-sH_l z?-%xa(OXJudoVzJmsZx#I=HV1W+7B3p^^+XW->UL!0yw zO5iu2MI+G8@6x}8`$$5E+`E)NA9ddU=k{Nh=I!~-$BWZ*^S7(pyHRHp zQ->Gf^+mU+hHF1utA^{Vo2nXkziFg}l1O4g4EI1_`L#cw_f^Wp1h|L`U%w`d{pE*t zQOzn*_$6N-d5ezHQ}>k2%(T!i)q@VY(3$S3fD;cRUm3K8wOD%e49z-nD#qnmoIVA1OS5Y#rhXRZmGM z{C3QkB~#3K?BV)P&KpZ?Okc`<)c)bki;XYQoy? z|D;xAMvp;E_D=>Py_ci6>AU&J~7{0dE|#;>A;mvXW~45e9CX#@**1X6@1+jM*BMTZV5jbMOAv~yRn?gSlg&; zX-cH7%(&2sW%U+ju3w?mcBXdK665q)pR0*YLPf2G#Ml}l<3<+HFmkArnFZ@_e-;ne znGTBaKf9X@;4E;Rh>esAKFd+)d$L3ui8Y5D#+c(xT)ygrmqwj#Rs_Zbm{UAi>ThzW z5`&dteBmROA`*Wdm2+|`?rMW!%3;Ep!W(}+C2abun7*}U{M1zoWlex#%JC8gli|27 z(c==L6h>@?ru^eeI$kf-Rj#H)il$u4ZhSm*tmsonf}m_HkQ(IqHs#tg^Sy>jBXo{L zCJd~gypkos9j`#D3AbgzwJFxrX$kqDbObw{!Ul%AtJt>GBK27&pxO$eDxX00{hDdcWVvtNqJ2OISUjTEB}f@u$Yg0I_HFhNCn+^e#n)`I8EAg4Rfs= zg65$kjHO^9moCm1Qq>Y3MH4helihW<=9P^sui5%w+46KF zi)SjeiUj6q9Nsoy@eBO{&UXQo${jm37$;OE%9XO00B6j*h$Q zPzeJsMqbanIl+sms@=EYgsDuE-s4gFcxP{SBI)U|KuSiKkDf`y%&BfXI zhLt%pGJ&@u>M=hQ-2y!}JnA36?jQAS`tZ}y@js7`j{cFLioQFa(f1q#RFJ$G|1Bk3 zQhmy$Y+CMVPhsw*MA`W|CaEBg*p_3q<$$QX4_oDU9D|MW%d9~;D7cC}j)(GAX8I+6 zOI%bhjW8&}pa_F*7Y04&Vk5P1vZUmW08fsc)dW2SxXAx85%OH!mD>b3Q&qZ6n^qH6 zsS9gXz?q8Xj46rcAvKxnSg=-2`Z7Zgp&A7wRNzaC&tl$NXoY;W0PK_nVG$gdJa{28 zr~W+50KG4HIJ5nL9@IfyUV-?-q~8LVGFS)U280`M!-sGK!VP-@H+1tOc4udel7dRm(O{33YA}cYODo0POn8TXut)jJd6JuU!#Cctli+;!VnwoW^1z>Zt zR?Jx$_MgkFvu*WO4f{dyRaTqzL$S;DfQ(v^2jc9Av+o1WUWTWqm<(d+h@~TzzL!|K z+>627?bxJA=;=f9214tH5>1?S=~f~3oQCZ_fVdpnZpltcEH)h$ITrsZe&vB}21Y+4 zZNo}#C07WRWOSrRD)GenyDCYr)|N^sS{4FZ80Wg~ek$2VWkk2LpDKeG6-jgxGZ}8; z;Au90)o}3KW`^8k$ZHm}oe|y$aww01-q1K!=yEEa_53}n!pn2=ZvwsN-a23pkgA_J zXlH|VeZ5@E5rk^F+BH-tpAz)DE%6A%^Jnl|{8E*684_=0D8fm~{;m%fHJ_-;2anAr zs)FtDLIaK0T+HR^I=)wMoTqX;$i01tOT1D!A#{$=`CdckQn6FW1rc;a&=Eo3M+9AX zb?f8jGS+F==y^@#JkKoR=XIpCP4ryXF2v3oMj>VK-|dP+S~mdq$pes-z}p>!AYuGB z#(!h{_m3I>y<&*2ojs>T7r+4=dcm2k5g|KKVM56Tjt)T0rjBUEr%-A^-9{)Ga|Xz z%(Y%WaiF(6|E$femf@Rvi?g6l_5KQXQLg#LMGmL$3x*h70)++I-2Rdll?fo2vnefV zxb~pcYQI9omrWvTVE)G)SvutLM$6vmxY@OGACe7l*+SBlk+W0wQ6ly_Ei#$X256$jAqzwG{d!6N>sYM z<%$E3{_T6{Lfntrj98Ew)Pj6^C7%i5O*U;XHOC{#$K>LHn9oVFg&YTdq z@WV$nx8A5zV@epv@DT9{EuNaM@S7QBs=m|A{lzKvG>AIG{biBH=>HQR!^)xlEj@yz zp#M8KgtZC!UqN+K^#4#je7an68PL!Xj*f72gzupvTr4d>v2xQUzneP=raWlbX)v*R zv)i5XFelcv&0g~@*EkFo?~PVieMYhR2$+yDCt1Vhb7MvNR(BJ(ct!Kqaf=uE)a!oO zYDpgG5=WQ#KD)#fuM8gSLMRlW&^?1fv&=nJD4KMt?C`_Nh=^_t5mowfo2Y22Of3LL z9d8UOm40qyO2k@o408}Yh}39Bd7yTk84dI8Y$@N)dgs0{;ST13!aPuz2kIH}K&6>E zS0{nO#GRP96BBnL&WAYPzTpx5Vl8m61tPn zorG$+nL7!wXT+ZO8hc(!7=idR;?Iab?-~A_94MagR#kF*Bx+G4BE@Eoy_nPH@OZVbn*tUEK!qaTG!>6I0T!n54mQ43h+mX_ z0RfTniVb2aR~W&!16wvdZ+Mq*Q18fLtuLh3!7K|75bM}w^POQ~w9 zBj&C7>S0|qeEK{QwbZc~xv2b3?>Y$j0iJqa`Z7ZCXWn<6OMSg)(SuG2fzhJR^_lmj z^-jx-BONB+i4|JgjHF<>xSI?rb>@=#=8_VuWifi`L!3#9 z9~b6E+C^?8^kbnPYajer&67W?dA!lLg}yEHZS9|LD_-i?aE-{4c%)+Ak5pt`xn)0> zq}l>cm*%Rf>7oik8k-(0byc@h#``Ajr>9E5N@vfzCkwmL`DSEwI1o z=?p~obstlCi~xT09ZzSp=BwM4zNMXH8%N&~`j&Q;iKID3-v0QXDtFng5%kU6WzQcC zWNrggf8t1>>L8#d?y@o`+aV<%&&i`(5_i+GWc?UJhcR>*Lx(YR7(=(>6gs*_(KULo pu2JQ?yie>LeSsHVc;ST?UU=b!7hZVbh1ZXE{U7OH$f*GE004S1f*Sw; From 0930d112501c87fd6db15a5146c81d01065ba4e8 Mon Sep 17 00:00:00 2001 From: IronPan Date: Wed, 23 Oct 2019 20:39:34 -0700 Subject: [PATCH 3/3] Update sample_config.json --- backend/src/apiserver/config/sample_config.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/backend/src/apiserver/config/sample_config.json b/backend/src/apiserver/config/sample_config.json index bfd64662901..59390f49c94 100644 --- a/backend/src/apiserver/config/sample_config.json +++ b/backend/src/apiserver/config/sample_config.json @@ -4,6 +4,11 @@ "description": "[GCP Permission requirements](https://github.com/kubeflow/pipelines/tree/master/samples/core/xgboost_training_cm#requirements) A trainer that does end-to-end distributed training for XGBoost models. [source code](https://github.com/kubeflow/pipelines/tree/master/samples/core/xgboost_training_cm)", "file": "/samples/core/xgboost_training_cm/xgboost_training_cm.py.tar.gz" }, + { + "name": "[Sample] Unified DSL - Taxi Tip Prediction Model Trainer", + "description": "[GCP Permission requirements](https://github.com/kubeflow/pipelines/blob/master/samples/contrib/parameterized_tfx_oss#permission). Example pipeline that does classification with model analysis based on a public tax cab BigQuery dataset. [source code](https://github.com/kubeflow/pipelines/tree/master/samples/contrib/parameterized_tfx_oss)", + "file": "/samples/contrib/parameterized_tfx_oss/parameterized_tfx_oss.tar.gz" + }, { "name": "[Sample] Basic - Sequential execution", "description": "A pipeline with two sequential steps. [source code](https://github.com/kubeflow/pipelines/blob/master/samples/core/sequential/sequential.py)", @@ -23,10 +28,5 @@ "name": "[Sample] Basic - Exit Handler", "description": "A pipeline that downloads a message and prints it out. Exit Handler will run at the end. [source code](https://github.com/kubeflow/pipelines/blob/master/samples/core/exit_handler/exit_handler.py)", "file": "/samples/core/exit_handler/exit_handler.py.tar.gz" - }, - { - "name": "[Sample] Unified DSL - Taxi Tip Prediction Model Trainer", - "description": "[GCP Permission requirements](https://github.com/kubeflow/pipelines/blob/master/samples/contrib/parameterized_tfx_oss#permission). Example pipeline that does classification with model analysis based on a public tax cab BigQuery dataset. [source code](https://github.com/kubeflow/pipelines/tree/master/samples/contrib/parameterized_tfx_oss)", - "file": "/samples/contrib/parameterized_tfx_oss/parameterized_tfx_oss.tar.gz" } ]