From ecc64778566748aa5814aebac8fc187cc14054e4 Mon Sep 17 00:00:00 2001 From: jtyoung84 <104453205+jtyoung84@users.noreply.github.com> Date: Thu, 7 Mar 2024 11:23:20 -0800 Subject: [PATCH] feat: adds default buckets (#81) * feat: adds default buckets * feat: removes unused imports --- src/aind_data_transfer_service/__init__.py | 7 +++++ .../configs/job_configs.py | 24 ++++++++++++++++-- .../configs/job_upload_template.py | 16 +----------- src/aind_data_transfer_service/server.py | 3 +-- tests/resources/job_upload_template.xlsx | Bin 5564 -> 9033 bytes tests/resources/sample.csv | 4 +-- tests/resources/sample.xlsx | Bin 9090 -> 9133 bytes tests/resources/sample_alt_modality_case.csv | 4 +-- tests/test_configs.py | 8 +++--- tests/test_server.py | 4 +-- 10 files changed, 41 insertions(+), 29 deletions(-) diff --git a/src/aind_data_transfer_service/__init__.py b/src/aind_data_transfer_service/__init__.py index 0892676..d41314b 100644 --- a/src/aind_data_transfer_service/__init__.py +++ b/src/aind_data_transfer_service/__init__.py @@ -1,2 +1,9 @@ """Init package""" +import os + __version__ = "0.8.4" + +# Global constants +OPEN_DATA_BUCKET_NAME = os.getenv("OPEN_DATA_BUCKET_NAME", "open") +PRIVATE_BUCKET_NAME = os.getenv("PRIVATE_BUCKET_NAME", "private") +SCRATCH_BUCKET_NAME = os.getenv("SCRATCH_BUCKET_NAME", "scratch") diff --git a/src/aind_data_transfer_service/configs/job_configs.py b/src/aind_data_transfer_service/configs/job_configs.py index 776ed5e..8c2a91a 100644 --- a/src/aind_data_transfer_service/configs/job_configs.py +++ b/src/aind_data_transfer_service/configs/job_configs.py @@ -19,6 +19,12 @@ ) from pydantic_settings import BaseSettings +from aind_data_transfer_service import ( + OPEN_DATA_BUCKET_NAME, + PRIVATE_BUCKET_NAME, + SCRATCH_BUCKET_NAME, +) + class ModalityConfigs(BaseSettings): """Class to contain configs for each modality type""" @@ -125,10 +131,11 @@ class BasicUploadJobConfigs(BaseSettings): aws_param_store_name: Optional[str] = Field(None) - s3_bucket: str = Field( - ..., + s3_bucket: Optional[str] = Field( + None, description="Bucket where data will be uploaded", title="S3 Bucket", + validate_default=True, ) platform: Platform.ONE_OF = Field( ..., description="Platform", title="Platform" @@ -208,6 +215,19 @@ def s3_prefix(self): creation_datetime=self.acq_datetime, ) + @field_validator("s3_bucket", mode="before") + def map_bucket(cls, bucket: Optional[str]) -> Optional[str]: + """We're adding a policy that data uploaded through the service can + only land in a handful of buckets. As default, things will be + stored in the private bucket""" + if bucket is not None and bucket in [ + OPEN_DATA_BUCKET_NAME, + SCRATCH_BUCKET_NAME, + ]: + return bucket + else: + return PRIVATE_BUCKET_NAME + @field_validator("platform", mode="before") def parse_platform_string( cls, input_platform: Union[str, dict, Platform] diff --git a/src/aind_data_transfer_service/configs/job_upload_template.py b/src/aind_data_transfer_service/configs/job_upload_template.py index fc53915..867721b 100644 --- a/src/aind_data_transfer_service/configs/job_upload_template.py +++ b/src/aind_data_transfer_service/configs/job_upload_template.py @@ -18,7 +18,6 @@ class JobUploadTemplate: "platform", "acq_datetime", "subject_id", - "s3_bucket", "modality0", "modality0.source", "modality1", @@ -29,7 +28,6 @@ class JobUploadTemplate: Platform.BEHAVIOR.abbreviation, datetime.datetime(2023, 10, 4, 4, 0, 0), "123456", - "aind-behavior-data", Modality.BEHAVIOR_VIDEOS.abbreviation, "/allen/aind/stage/fake/dir", Modality.BEHAVIOR.abbreviation, @@ -39,7 +37,6 @@ class JobUploadTemplate: Platform.SMARTSPIM.abbreviation, datetime.datetime(2023, 3, 4, 16, 30, 0), "654321", - "aind-open-data", Modality.SPIM.abbreviation, "/allen/aind/stage/fake/dir", ], @@ -47,7 +44,6 @@ class JobUploadTemplate: Platform.ECEPHYS.abbreviation, datetime.datetime(2023, 1, 30, 19, 1, 0), "654321", - "aind-ephys-data", Modality.ECEPHYS.abbreviation, "/allen/aind/stage/fake/dir", Modality.BEHAVIOR_VIDEOS.abbreviation, @@ -65,16 +61,6 @@ class JobUploadTemplate: "options": [m().abbreviation for m in Modality._ALL], "ranges": ["E2:E20", "G2:G20"], }, - { - "name": "s3_bucket", - "options": [ - "aind-ephys-data", - "aind-ophys-data", - "aind-behavior-data", - "aind-private-data", - ], - "ranges": ["D2:D20"], - }, ] @staticmethod @@ -104,7 +90,7 @@ def create_job_template(): worksheet.add_data_validation(dv) # formatting bold = Font(bold=True) - for header in worksheet["A1:H1"]: + for header in worksheet["A1:G1"]: for cell in header: cell.font = bold worksheet.column_dimensions[ diff --git a/src/aind_data_transfer_service/server.py b/src/aind_data_transfer_service/server.py index 4489cff..8452e26 100644 --- a/src/aind_data_transfer_service/server.py +++ b/src/aind_data_transfer_service/server.py @@ -15,6 +15,7 @@ from starlette.applications import Starlette from starlette.routing import Route +from aind_data_transfer_service import OPEN_DATA_BUCKET_NAME from aind_data_transfer_service.configs.job_configs import ( BasicUploadJobConfigs, HpcJobConfigs, @@ -48,8 +49,6 @@ # OPEN_DATA_AWS_SECRET_ACCESS_KEY # OPEN_DATA_AWS_ACCESS_KEY_ID -OPEN_DATA_BUCKET_NAME = os.getenv("OPEN_DATA_BUCKET_NAME", "aind-open-data") - async def validate_csv(request: Request): """Validate a csv or xlsx file. Return parsed contents as json.""" diff --git a/tests/resources/job_upload_template.xlsx b/tests/resources/job_upload_template.xlsx index 14d0f79cc2d9f99198dd7d7093399f3e0016547f..227cf26454bf72c1b18d8ada7db0360a695a1d84 100644 GIT binary patch literal 9033 zcmeHN1y`Hd)(-9v+_kv7mr|g`DaDEfcb7CkaSEYOG*~Hx;-$C~+_gxt;!s?Q7KZ{~ zrZacVOlQ7daPP^=IV&sgvsbdue)it)b6!nV6jTxbIsg*@05Ae9_S5Z5kN|)lz24?}~4OcH8Y0ZLQTXqz;^8~3_SEn$V_hbqNXcBEu7L96Mr zy?TU8J@vSSh6NIRbM0R>^cD%l2R@DiS4W8_9PBxKO^#Q2lu=(Z`OcF;O!71t^|d)H zwCX*sQdCH%8!vJg@!$Gj$EyN#(pna*<)Ux zXkImllG>T_7-h%?Glq;!12i39SCf?px{UaDJs{0d`*hgBuyDp7-vYphrWKec-xabOBz3y{85ZG)JGS0Bxi)%4aaSA!2UKyX8-WL zuYKE6zPq2-jb82_C2t1`-l78lcXucN&A-vIMwkEI2_n|i5Uj&N(9+by+R2lT_qX|f zbo?*o;9vfFF--kQCqFLqK!DI3qfS7O@;pIJbPhB1LMs~$P>1Rz zieg}Mz>k5s1?lLWKE|^Z;o{dsBvSWky-J`dSMJX-I9WZCAG;SXcM(D+Pbbe(6;*uL zAkH?P`Du5%r±^Jt)C_fvxnd(;gUEiOutCgrsEN}%O+VwpP$ zBWZzC$@$xO4_-;A?T)3A_IX;^&XoG~IWeAIQt4{jN!XWLWOzt2`IuTcwH+&@wG!X> z@o2p3(_#`Px)2;u>}AP34K%D4KKhjI(MU=dNF{z($s1}?2!1VQK! z6e2))#@mU{%hkit!qwIBx4M<9?Rh$JW}e^nEavX?rgXsme!W!d^ctlB7Qa@}8)$S8Q5IYy*RbbgV4};TT)5Qs+qw_1MLvtdt@N<@}GAOVrvdY z#_N&g)#lt980b#t=;XZWcRi|0WGIJd%5CAtL@7GCdk^v{4r?9Mf0z^P7<8YlKEe$q&9u_#)iah zZisa*p6${={P*=DYkWo{j9^2w8?t4VJEx-`_C_5*sryzI+Te;M$xh5a%itTRM%jfJ!lwB(gQYO-)oDb|a z==mT!zlM*Vd*c6VSpTO8AR&4(1TFr% zN3oW=Y8OAS8Rs^f-zyD5N;m`NW7^gIMu6F0%`wNyC=_tAM9tX%(*L5$hvEwLIT-4N zoQvSEpb($7a}`7qp}rt?I*>wh9XcLD!L-;bp$kz&BPQD2`Bt=tga6i(xLyX<$DKhU zzH!0MLK&Wu&!litKL!Eys>t-p7239-}v3>iSt#N2a4@9u>I08k*V_#+2; z+F4tJJ^6kv0>350cM0lGR{4RyRSS$y=UJ2t zQ8rdT+R0;5dDkCx+$417?a?|KS={DhudaM-{uqnG+|izl<>p3|{zqmPhKoaKaWKrK zef)ZRz1zxpVXO`i%+P)Tt7QbES(;R5*3-}8<36VPCQNX#O*ZI-@0}hn78zZ+3$^FP zTOh>t!tWG~CgHEjo-*seUy(fuy1dqd1UwmEp?SCAGUJ6}{yhAZ3MXYPmP!O!_i=}Q zFIU)e*Y5)4R;!0}%bqnU%D2QUJYiLfqUNuxYCm08|B!<8!o}k>zCFN6h)sQ&6h-Z= z5Z#iZ72t0c9aCQ9+Tns<4mU2{vIdRa&1{OadC`UMitkz;fF(b&?7J05ircAN#gWHa z4&*0HJP2eiz9COTHE9k(CIpChs95;wec@g0vISc!o!HAUdl0T+2ZU&`q%S+&@!4la zAn#@#PA(P=Y0KYvh`oMb>mnoL0UIN_=U%A%F6>IC$rr-ql9m}U$&fH5IL)9DuD&s8Af{* zkRtrm+>~f6hn-Tu3}tu_r?pn%&#KGv$4+K@Djv*Ni|7GYFdq zrF>N9d2#r>Gc8*7QN^RMux9MY{LJS`O*ke=x}2(Er3EOYhlxQz0!++jIGL0!d6YP1 zLz(7|pu9{Ps*491pOegz?crkRWK_>#NOz(4)blrR@F_nSY^ z+|`MCwRAZudGZ$E6*@&vn`V722m}lVHQV|h>n=4jy2a3U))v7l$o4!5 z(HZl?WbkZ>a^U24jS?(97Pdc{PdKO9{9)RL!6CE60aZXk+^#xVs@M7LRro%BwG3Jf zG0(n}$)z zt|HLW%BJaxz3cmPdY3Vzy6uu~;*ST0WcUhsxXvH$CZ#NkQrU>lrcWrMRc&gWx;MWz zm~{V!@j#udP8!y&ig|~o60RP!%2$@9cEy*emi0ik{`E62cZKiXNq16d>t2$r=0XqO zrdy$G1=!SZJm7yCP`IDkG)66p<%3yX99K$932J9|nO<7DBL$|gu=-uV=5lZ+b7VtT zA~sx?G%eOEnt7z(R0Qx(|G+S!`#=lnBE?HUZZP>TUu$hV|`zE8MkU>20|D5}XTqV5*cDv>lF zK@KT|UiAzdQVpd26|%wE_M@u7i!az`(UpQs#y+^GWvd4_$%U_}7#iw0v7uj)(>n2- zul^WugEqX_dpPi!kr4L1m{f=F8L~oUoTBh+qxSTGgRvJuxg7?flp{)9Z-A7>bAETt zMc(o(%q*B^wjL;TAqpS;EUh0w%cItW-LKF$h6p-nW77|N}^*E!GNXEHk zDK&z3_m<K8pydxEn*G zwz6kbH6^+juCipNrfpNRM(G-z?bLDIVq}NSTtknCx~Vlg>_iP^9mCIFpq)}$_07bP zx8NV1svy13VW6@Z%`?FD$xrS!j6CohMF~O|EVhGAFtf?i$dP!tNdie6(Y;@O9iwg} z83WmpwatO+IY@=|sYaiea3)?hKnuCwaZ?fpX1 zxqWm^H6opu#hY^9RNuSqNPO-d@Y`#Qk;aDW(tY&1%Z5holo3PcO*FE%Yd)q<4QpWg z1nJx;+5)V5VtLyMEHq?K3CPq@T=U^GpA*>*YuNYkdC8`CABGgg7O3D8y_1gxiQF^g z#u=+nkVxU5OSKmAgArE`i7RIfi6)qXMsRECl!_rfRw+{hRm)LPsU(&{ff|%0_QE;%k{gDmHaXYpTa|^xF~#B=kve5s{g?uMje}q?yU&^0XHR&P~F! zE}JG}KE6z}26`GA$yqsipNq@BppdLIk>Z*chn3mmo1`=xb#g-+vox`&&pxnLVNKPG zrQMbncc_QeT!(aM(DsGt%;~d=3J(@Bq3YW}-R|8v&vD?H*8!bgaqK@0hdInR!-J+KBXiNin|e=(M>F4 z^5RFcD2(%K*iqn?T4Y|Gjx6kTEIeOi-g>pYAiLmi`rddC(a!rfS{cHxT8AW5J99M&u>ysWg!J|MA}T-mK;Hh9>lprdS@YE*%-7FXU<8jR;(i@_eODK?qQ zj)Lwbcsswz^b)Uv~Nso8$4paYv8$2>FIHFy0Cc}1=63ccwRq#`i2U2 zW)-<}W+&N4(fh`<)2T_0JC4 z|2$O*9R&ag!T_%I&z~BC?t<$$KQK^nC9wQQhD86{F+F7l0U<`zozcFU1OYwIS;Fk z3OZc6q((BUCl2s_w%azZwV=cFG`~qNW43H77&}gZE@wiHdM%T=O^fZRPNm6eBzBAF ze9@$iN5w&R1vyw(cuN`o9R&ZB?CN0FF@5HWM{v8Qu{(s3w@cuGz*;ZPcM_F;fgEh{ zO!L4x*EVZ3-KDhslZ?-c6b-Q}RwqGV^kZ;|IvPmSCZBx$qgl;MecUCdn;kI9QVj&3 zQ{V}ghM3{gwePPz#;H`y&2F!W7Ynj*&%W2Hm zFINPkLi!VjLuO6{u_BoxpN;o_m*jxOAM^`_J1Aur(2p!$S}nt^LW)qHVL*Gd7_edj zF+*tHFnnko{()e`zYs=eO;4Bt(cuXp=tS}dVeHI3tgW=c9`-J_KY0U=S6A(nMci(N z+%fJpr!(KkGAk2=e0>Nk+!Wl`NSoYsX%ON#TWZWBVXo=*dGBMokz;rR=atg5qV_9^ zv@aF>E=lpttLDl(FhjZ^3az=F*Sl~sUccn+bz`JK<=UxnkzQ^fs@VceJ_DNJ$213| zRpDI4hsTCOjhNhN6Mb4x!@m@%g}1Ag45SXAyPl)mPY<)mdK>Aitvv3<{-GE<_v6>x z^yd@T(GM9bVtS&&NBG7nD{Rz^mD2PjHpJQ!JAtRVYu)rO5D zF_Wt2Yi4+8be3(U_XxHj=wn6@RHGlP{AZj2d;9}ed+X0>DvSqh*C)yDp2PB6?%$>& zjg6<7{;MY~fzV}jgeNT!o+L!*Jyx!knjWrho_vmZC7rKm|r=5oQ zjVnQsYHkB~{2$Z=2FtGH6P9O|%iEv^bw`bhUl?4jR!sW8s_$7REiYwZ#M~-!zG|Z$ z&AF5CQFc8T5aA?(am&F%*BY3ySOnC`4mw){=jm}{G4}eunp#l7td*068HJT?yM!_+ zngd%RR}d1>Kg%DMWsI61A_i{|F$nx?44S*S{VxO&vG>P#2m8zE=l$o{m{|h?a?)MM z3VCGu)V5>#aL5{Ec5<05>=g`|3vvu^kZGxHm@fAJA}2>K&zPTJYz1`U2I-To*!%c` zc?Fq?RaPy7J7l%Ie&J_AZpYo4}saA}3Fl@EE^q)ei zD!wMs@R5%5w6(A)8l`hdxJ-6X5j=A$=2|68{6SYB#SgOcTqSat94kgs6|mlt5wEq3 zZ>9>x{{Spe9!9k&)({o;-=ogWHMMxrNfo52OorRpLgUz5BQ-g0($?KlE<_>gT@ech zuOZPF)hnN19D&v5g>aithO<|%!={$r5L}u2A9PU5hWm1F+2;t2d#apqS8zc$W)8bBuhY2eQ``d8?$ zEySNtA*$b@zcv)VTKIeE{1XoV_#@<{e-zSR;eXG1e}yB_{RRF<4%AddL+ma9fQ@(s MA@XDu!*8Gd592AgasU7T literal 5564 zcmZ`-1yodP*B(k3ngOImBm^0{LsB}G8W>6eX$EN!1Z3zOS|lVD5Qa_xsgWM(4hbpg z7LXtBUH8BD@_pZ#b>5k?)_L~4?_PU9d+$eE4GWtB007_tz;VhDMWa^2*Vj+w*Ei|) zZ3(l`c7Zv&@;r5R=Js-Q(1_6>XyYY)bfvmJ>!i1Pm+$UK?n3uVWf9>xOL>pr3tyeq z*lEc*-lzu39!LkXr%*`8`&VS$N`M*y#di7OLK-DjT3g5+nE~%e@=5CsmNL38Smu=K z0t0U^7=BcXuhM@KTI5(St_@C2cT2J3V?Rx?KZpq)|HN1WJ9~=LlJFBp`(OJ7*qXR( zT<^0N0{|fV=YB0^!coR843Z|5Zpxl-+Zh*(N)RZmk>(qbUgpP1-@nns zk_t9&G6zyD5`)Wy(}behB>PZ5as)*=+m#&d2g<%n28(HG-)5bG6xa-B5$W(J_}M)# z+HA;yYR;5B1|nk{S8dNey$G+C@7d^i*QT82_wh?)TRn{PRR1G)WedLL?RMJ>*qUaqnUVzf_Fpq6iP0#AX zZ5^_yl`PHuOu;rXUv(cF!OZSftuP9lQugXbTjgSq8_2TNhgI#LIdoo3%}H6I*Tot5 z1D$y3Ab0%-VDFbTjl#a^wsm~AvzFGk|1vnF9Pql^FA=9f(4i$y7Fd;Q`_*#dc&ZdpP!z4BHGJYX zzxmcC9X}eXHEi!WI~)^^D*5U~)V!w^+FlxtD*o!WY0Gy1veZ6r{kDyq;?kxMh_ZS` zX80}u;_NZjH-wdJ%JI`Zk#>@yaaPuN7qTrDPWB7ci9KdJM3XG2=#mkKLktk<7*yBx zMatGhEVRWB8OM#GIjXg3@zS~ikCa|BbL{-za za$Z2NFEdTsRa)0x_ddU@dlfP&-~f&}Q5vCAT_$0BFTKt$h?|>c2yBAu+hrAn3QB#r z!;g=%SAR!p#m5U@BpLNdyY%%-jH|&Zk+$N~*Pq4wx&6@jJN1VTvxjO$nEk9$_t}nXC1da)@tj|9#ZKN{ywth~sz>#T&T+FxQ)B-DIS?Vq# zdv+gm0M{&`tiWE~VJndcP^KK6)4I-)YSQ=$vYx$861u#O%yyFkqt7JGR;&661mtU7 z)HZ4oo2%JGRS(PioHg(C5=vN0#xvA-zGDzNKQ{4LX3po;Ns+Ikm21XwY{-}q7tKsX z)Z-67iG#JDt#^9SE{aWJ*ZZX!IId0%4xX>JJ}%4MkxXXz61Jbiw4EK2if)$=^?qrV zYMq4A7Nl@=_{aiQtTnG2-vd)DSekRa!wo_{#@v6RXU+#jVR6Rg_^L-KtU0fy7@I8( z#xG45SXDbAIXr0`?>!M~vrQ$pF7BDLDsRo7v3O z+I^fdxb9y~n~prJqI#%!ko#ugPGzX;Tc5f%g^yQ|u{&q{^J31^;&WK0-^s!9d9Pai zA|e9Fik~$%Ahpg|(Xe zi7z?xYlo<;ts~~!R%8ptzY9o=jOpnW768CR0sv6|SwK8tF7~cAR#tAVJb!%sQAMf^ zrehXGNiUPw(YBZ>0Z!P;v;BHTf_TrZ8KG=1!l>>KOc1+WT_$JrVkR605AMn=R%q34 z(3?GICtt!PdkT&oPi1>%sraM-Yx?#pxCM(5xD$8?kNTo4m$2iAaEZbsiYVNwntmvu z$inwU-Vu7R7iSe8kBzKnst{GY*aGT!(7s`)6m;m4ik~u%4ds|x;!5^^4+tnQIRh$3 zk;_7!In&4VeHk$SW(&Ww7qpNd`cg!fUjjv3R8FY)9NG!zIL>zoTzJuePJOZhymw|O zDs+{l%Y>d+I}yl`hS$ntN(=lX0a^<3?`SZ8X6pGcQ#OoACFWi&pY=HVDq2UXZYd2# zOd!VWz4(faC6YkvYm7>~^q{>;pTVGJpof!OBhFM4aEsDrhs~{4#uFxYg70?yvqk0v zCO1#DcJ52FQn9HZAPb5PP0t`K4GFS8%1Ox#3DSxG;8&4E?*m*|7R^^KniZ~M_0kcZ z=84nVVMbgXu*!hcCIp4nR!;`+O7@kZ1F(po{eo~%)Un%pEb1U9!HQv`(XsY z+8~A;tQxE*N`n`-jHp^94uad9^2??dJP159XG&9w#lvUKSdZ^)Ga5CWzdBK!X1i)C zT@`ol^%{bIohq7cdOVIY-WX05if##ig3KAQcaHQ+SA@CuZgdwjS54a=G1AwL54Lcc zzGbuhUJcCl9z_U&&X1y=Ys4WP6WMqecSbsQZpB5~XU#@NWH361w2>DHJUs=gSvIF| zHx|kD(@2Ut#tA=r542u~XMpsUY}@%&1nS%5WbnK*fGCdDs4O}rh=)%OyVx3={w1^P z8~Ra8Mem~;b>e`8?U|~}%@XDh)z8n0}T zOf5eef-j0$uZpRMM`!R2=4igVEqN>h7}6~qv4pJ>Kt*V^@-6PnUI`n6ss=c~1Voh=Hdm@1L-eYAT4 z1;dn+?Vfq4%Uzb5F`vRajfnz&#qk*m>5`fGQN~_j{hKrj+wCesF#&*Q1ONchpQPdH z=IvnR`n%tp)z^r9%S(F68Frq`QpNr;Kpt^V&#pMhgQO{Kf+hJvHrkN&^61DMcDIqH zNPjE;5pc=-(;Jb@a^=D4O7mDHl!IHm+>w=URZXK-{40OU_`@ByFNyL-jx6O)w!|v) zG>SGPEFAIuc1f)9&Xfkhw9A@PQd66bsq5Q}F|AdoEr=X?msh4PvM08hy1g_WQUcVFvqh1R;@4)^d_< z?7sF$Q2LbNs{U6IcfO1w^Pjv=F@WE{`R*Fd9HJLJE)1t*gE*C=H+O<+ezcY)XQ7S? zGqBCSjbGjMO!%O!43ql3_A{oO*2<_TGc9##a_5&`;bXIrz1;*<1rti(qskaC8Z`Ke)`rlZkcJ zJMB45LzBB7R55z{X>uzSFzf*~aT-Cz<@Nv%;peuAi)X@Lxp`uUjP~2Ya#(qONXm5G z!Mu+~V92;);5y=qKfa6EjguQvHbn6=>cOMfGt9(Qc5L|wby)IKx};Xzbt<2*Awe`5 zp3Eb~v>a0U2^_p63^{FfjmyZx(Q~l3H=1?0Wn<&UoK&l{8S(i#Qg2^J?RC%s%v`J- zTzR;EuNm=bPMCazGDwh>nR;OdCf~=2GWjrqy71N9HysFzj>F^ViO+X!$keVfGr5o? zg6}-&U8nOMP`n++llnp>fC_sK^UN1HkOx9)GF;h$abJ%nNWtF^Si88|oyKa_T`yD) z@e!L1u2oi^_!LJNr={&_%&PK6lP>}U)UNtx(D);18OV=6F5V067W5q)-HpppQg8+z zBJ$KDJra>pnX}D#2l*Dys=Vgct`5F_pm=!JlEIrFJ~XZXWP_ER-I$b@%j~y#ucu`8 z>w{jG^xp&F?|S|W2DAU2oY8TEn%8r(f`5Jw?pLSRD(ai2sR1f@r1CG(MxNEl9Z5@0 zWyt#~9O{d~_rHvfbvK!&hmlC+$R9tAK5*sM;~eTgm@(jjY#~2jQ9UOr9|alBt&Rvn zS|f_)UMKh(QB9y2EIP`hnG0p-tS45KxX6ht#EsCOHFvBjj)(zU7c;A+0xIqGSXCZ#i&2#BEDjpDp(hiDtQ$vx<|t6`;qa3yd}qj z73i{Yo5Qhit5H0ATt54z@y`m)yntR18P6@-*vj<;f1iBmP?MQ0gL)yfQe6qt!r|uP zjYGZ-k%QA+A8`D-EBq}fK;>ahZdOiiX1dKRAR2%VD$FV zS~n27J5sE@JC0FPjRNMOZUv-sDGhc~3hu={y_oBFGLNfQULJA8C!4Is_Fv6#STBz3 zl8fsy7F&i}ZcW~Y&%Ge}sV}N^%ADu1i``k)$9UkGb^j7d>rIkzGVdHZkZ9`V0e{3h@w z5B?3Dx~9b&tauZAGtd79i(IRZ*Wmw=^*8O@Or^i=(1ZSKVri@4T#I=C0Py-FbLYC0v5=vHc{D$UwmDN^Nv39NuK+cc@F{r4S{khaTom34~}JN7v| ze$F}1F{d9lb!9v%-wgEcA7%4RtQH*9D=F=>80`{%>y znu~{f&L0c_Xyni=lv+iWCDVe{RG#n^Yrv9oQBws=v9#n$Fq+ANF;!QVjj?alv~0{R zLL`pC;CWuAEa7Qevqo(}1goe5`m!ijvPY}aW8i8k9@-U3dA$ON`?4z4QwM4qbsC*7 z8!qU51z~T9XKa52e0}&441?XD1g3+d>M|8v@?4z&w6!ID5I?~dA#YN!In2RvNUkN= zW2tMnXC?f1vhw#>{tFnve~d;zTAfxRNZvcqygfJDJUh+Hirs9bYtYr|6RmZnD$}T_ zRFkY!EE}2wSn@S{vmx5mxGgKN+5=+yrggTL^OZ0%meYT>QgaY;4-cK;`=RY=PY}ka zQcN^dv6wdsbhwM_Pc_$J#{~z48#DH;Ed_(d0)0WOFw!)l_wtIW!e~V`G~;OTSwi@W zue>>V;%&2BTnYY_rAjWKuQHTTMUfJ;kvS6xTps~>p&%D>0noe^DU*v=I;MwB^BYR0 zsZNgt`PqNQu>JpYt{@Z<^=iFR!hrwh$puvGnm$7PKqc9(z0aXQ#JOuy5s|y+$=D-t zLPp3Q_!H!Y0Y(7{V-)zYLu|))u;+h=V1yr~oVKdiNkc;qO&4@HHme~u9 zXT19$J^wWHw(mM%eF@%_Y%O0zq?g9#7HYuPLlb|65Yy9})w(m?mYFIb#Lx-AW4re+ zE0+cECAiy!R?)nM=JRM|2Z8Iv2*<9AT%5QlbjJkwHc1>OOkC_j___eE2z8e&5bI{6 z=^Er(U%&)zbv<_}HVqN!GsJmDI)whQlvXJ;gq}LbkYGF19&Dp7r0QnSVhQw!i$?<- zx(I(y91pocXd_^2qu5RCi8q)eP(EE9{j!Mu#+BK%MAW~9G|PetH5W8}fOpr7j64nKeAg%w!c(~PPXI!^a- zw~q8U-R3K%Oia+CiA=7>9NkE z%W*O~-hVWkjsBZYzVt^}zf7pr6TYQ!iUs0Cslp-PJ&=p;s&i`OXIQq z{WJDIe>!Ou9l&{MO-a&GLLxB2*3xWC^6Te1{z4)jxT(3a21;@UPo5T^Kh2NU?R*Om zA|x1Jk~V~{pwx>NH1{Lhfx(@|I?V$v-G7$)&H=B7OIpz^NhVZtX~>08xJMYaX{3Oa z6?+XPTp$h<4_Lo%W#_LnEgr#X?soew7FKtN*hnSAd4Q5gi{fQ#tm7MneV=H<>E%V@Bbl8%BX{b>_6}gm3^$FG{3R3H-Fgp zgRsdB)<2gk_OdR?Up8G=)i}vxwq#YzW^5cUC(9&e)pD9;`D&6+XaC4zUQ7JpzN^SV zQ<5qzo*AKw`QV@55{~{|90k7l0SZI_)JRFx+jC>>u0QyKsRW6BIM5Th2=V}5L8+2_ z%hBO~1`P~RfZi0|$5*xWVCe5Is(+#3dlg>o@fqZ{4JgY-Q-lqcDC+YHyg-klcAut; z{{R30|Nj600RR7#lv{3tFc3wzg1mqd8`C`GAT=?MZ6Jcwue6F$n(RJKRN=W-$_IbI znHhP`wQ%8k94FsTedR7cf7xZPw93M7e-80!w^7+m+kV)c?tTozc%JrZ^?$K0Rpo|f z?N=<0pF(`URx6iZE0+)d>>!y&c9hH@^O6QqPO|mP@~C5WVwO0G z#j}$QIgxBjP9)PLvj#{u8}st7O<_xt(n24Z?f@p$uzIm%^;kTwuIVTTfE3UXASL7h zriflwu$IuUnIf5?p)CxhoG3bgDJU>wmzasg@67;YGZGrQ79Kj;40KZVjeTzhhO!xe zY$i{?lLxbU54s8ob~O~-VgUdEmXkdi9e+)0!!Qhn?}Pn^;Ct#M9b=TlW$QZHLuuJZ zVW;9cTH~#;(?}`nzn`;DC_6X>Uo5@SD?ebR!}oF%27`qLh{P^fr6?ypkMbSN}@1sPY_=W;>ew&LUA*m zTra2d`_*!KJD#7yBk`MMc0WA<8O6$N-H)%H@e#2X|5e}s}BNoFe-*Xv zs0&;Z!HYc;`NBj2K{WfAhsE!H#B~JWso`-W1U6I*hL_7hqF8BGj$i4xSkpmR7 zJ03d&2`5q@keCAi0PvH4Ax;5-lg=S68>RVmmjeR;04fLo02lxO00000000000002A zlNKUH0qv7qB0vFblbs?U8+J7m++qO$0G0#*02BZK00000000000000xlh`6B8+a+z s_EG@=0Am6G01*HH00000000000002blNuvF0a%k+BO?YEApigX031J|)&Kwi delta 2513 zcmV;?2`=`nM}kMN$_9Un;qq{h0{{T%2mk;N0001ZY%h0ja%*C5Z)+}iZEUPnZEvGE z5dMDB{RhN%7n=Y9)N~aB=qk0VD$TZcQlym&rl11mY}2Hw_P_6#guZM}r)+nt)W)9S z@iQ~enA1<&rZyg#kQHxdCON^T!P<=HReL`(zpPT^n}$@h&1rwlTQ)PFm^43~{rSh~ zhKq+g&L0c_Xywc-mFhyvl3B?bDo=RFTCfycG*rP-+*`5}jOMarOf|J-W9(TCty*)3 z5Q<|kcu`ati+R>JtW`S@!D_02zAURwj%ZDG3|vFSL*Jo{HyuFSRduDF22j&zvhe)A z<$~VT5cZb1##Vp8&x3!0VX()Oz;tj_Q)PlnUZ@j*wsxcs;wRW5B z$!gY~)K$A)vX09&G*ymrswh%|wz6OXf!`y*t`#Igt^t?#B4cv>M#S{6V}3%$G}F1U zAUV4LcF%u5CJN#Z;V9KBB@B2zk1wEH*YpvJ2NKEmy?qYV@xMNkX^_k(BtZT&@R1YP z0gCL@NAZ-zNxJat2_fGg3gLwrr@boog3y>V(}C(Am@esdv=AKjRsMSZGv5CpJ^wI_ zwr@I8ef8Z`Y$IQ#qnF0^7Lw05pe)yiXrA6LH-mriw#ror9=?ykWw-YitL{r+OkBs) zjUsH#Y-T>s=iui|76>mK2RS%)P~b!ac{YhBlOT4m;|+|gH!VqgRH&7p*#msDmaK@V1SRbP0={p_ zMy?;>2~c!Wk|Ku$7)6oiAU94YE(mIoxalid9e(k_i^{i#w;5GEG@I_3;n9Nl9%P5{7?>}EI z=l`sdzsDo2|4gVg6kLykOs!ESXWx@i9u>2X3Nr!;CYgG&_5uI^9FxloA%9D6;xH72 z_apT`Sl%bj!+{bdR6z)}yK1JLy>V`cMQmqmQ$VBs_i=)Nso0=Ow&dq`?y*ntbdVkT z0Aqz#C62=gBT!t|LTyU?{?w&heBc$8e6OT9yRecgEPwqbY;u7O&t>wR zs7w5pPFGcxMp-}?vWq+YpYtJ}$v>pGvP5rkStx9{>OV|Nj600RR7#)LU-CFc5{| ztwdfxgD+_*GLZo1vW+N>)F-HLMCtD9qAH$qBp&hxd}9>~_LmEyW#eekvR&RWHt9?2C+fkZE zJ4rKWL7GKtq&c)!T7}k0dqwM|y)9Oz97_|krb!xJnrx|ww0CMEO;gMUkZQK((O(zu0f=+<6p#Ydy^!N!jfYiISxeR@22j^%Py+o6D z0cG5pGHLjE85=+@Bhk?21DxbCfLz8sF9YYe3_K2(Df8dSFSCyix(a`mX93Px0RRA< z1ONaO0001ZY%gTO1z zpB};+5}@U}<-DWVPBtj3H~*486RQB#uj*cF_I0x!8v# zw_v3@WyJDHzMQ^1`U#b?p-zYw2)6L#=Lx_lDEbfs0d;jmKlLO^qA>0vh_6O*!*0cM3aSo5fZNFKTA2?%=AM&Z&bcIC98?v3fpxOh1xgcwBG9}RQtt}%vRTGA zC~}@@!KzwAfsWwNqP)J!xuLwa@MLQP)=O|vAZbUrDbU_~Los%I4~n@YICK!VwSQI2 zhojwL!}tf=0nP}SV+EcG<{55*u^EWygcN*0nonDG0KqX-pg?L?IIG3!IhL$^FA-iOP&*)0m`jHr=GR zbT+56>3jS3ulsGpexObX{zuFRB{NDUBjOoQ=J?`Ul-~f8kpmQyX(JA^bRI1O35((K zaF7E40O*sdAx;E&3IG6;p9&h279tQECYgG&_5uI^90&ja7ytkO00000000000069$ zK_W!~*pr7MKmk{i$RZ#cmS+LZSpfh5odf^?6aWAK0000000000001VF10yE|2P`vO bMUyWh6#=o6MI$}|Et7;JBL?gs00000P`a-9 diff --git a/tests/resources/sample_alt_modality_case.csv b/tests/resources/sample_alt_modality_case.csv index b23dcad..1ce7623 100644 --- a/tests/resources/sample_alt_modality_case.csv +++ b/tests/resources/sample_alt_modality_case.csv @@ -1,4 +1,4 @@ modality0, modality0.source, modality1, modality1.source, s3-bucket, subject-id, platform, acq-datetime ecephys, dir/data_set_1, ,, some_bucket, 123454, ecephys, 2020-10-10 14:10:10 -behavior-videos, dir/data_set_2, MRI, dir/data_set_3, some_bucket2, 123456, BEHAVIOR, 10/13/2020 1:10:10 PM -behavior-videos, dir/data_set_2, BEHAVIOR_VIDEOS, dir/data_set_3, some_bucket2, 123456, BEHAVIOR, 10/13/2020 1:10:10 PM +behavior-videos, dir/data_set_2, MRI, dir/data_set_3, open, 123456, BEHAVIOR, 10/13/2020 1:10:10 PM +behavior-videos, dir/data_set_2, BEHAVIOR_VIDEOS, dir/data_set_3, scratch, 123456, BEHAVIOR, 10/13/2020 1:10:10 PM diff --git a/tests/test_configs.py b/tests/test_configs.py index deda2c0..f11cb56 100644 --- a/tests/test_configs.py +++ b/tests/test_configs.py @@ -27,7 +27,7 @@ class TestJobConfigs(unittest.TestCase): expected_job_configs = [ BasicUploadJobConfigs( aws_param_store_name="/some/param/store", - s3_bucket="some_bucket", + s3_bucket="private", platform=Platform.ECEPHYS, modalities=[ ModalityConfigs( @@ -50,7 +50,7 @@ class TestJobConfigs(unittest.TestCase): ), BasicUploadJobConfigs( aws_param_store_name="/some/param/store", - s3_bucket="some_bucket2", + s3_bucket="open", platform=Platform.BEHAVIOR, modalities=[ ModalityConfigs( @@ -80,7 +80,7 @@ class TestJobConfigs(unittest.TestCase): ), BasicUploadJobConfigs( aws_param_store_name="/some/param/store", - s3_bucket="some_bucket2", + s3_bucket="scratch", platform=Platform.BEHAVIOR, modalities=[ ModalityConfigs( @@ -308,7 +308,7 @@ def test_from_job_and_server_configs(self): " python -m aind_data_transfer.jobs.basic_job" " --json-args ' " '{"aws_param_store_name":"/some/param/store",' - '"s3_bucket":"some_bucket",' + '"s3_bucket":"private",' '"platform":{"name":"Electrophysiology platform",' '"abbreviation":"ecephys"},' '"modalities":[{"modality":' diff --git a/tests/test_server.py b/tests/test_server.py index 8cd7bde..d2a774e 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -268,7 +268,7 @@ def test_submit_hpc_jobs( { "hpc_settings": '{"qos":"production", "name": "job1"}', "upload_job_settings": ( - '{"s3_bucket": "some_bucket2", ' + '{"s3_bucket": "private", ' '"platform": {"name": "Behavior platform", ' '"abbreviation": "behavior"}, ' '"modalities": [' @@ -334,7 +334,7 @@ def test_submit_hpc_jobs_open_data( { "hpc_settings": '{"qos":"production", "name": "job1"}', "upload_job_settings": ( - '{"s3_bucket": "aind-open-data", ' + '{"s3_bucket": "open", ' '"platform": {"name": "Behavior platform", ' '"abbreviation": "behavior"}, ' '"modalities": ['