Skip to content

Commit f53a7cf

Browse files
vishalbolluvishalbollu
authored
Zip file limit (#457)
1 parent ee4e2fb commit f53a7cf

File tree

7 files changed

+78
-57
lines changed

7 files changed

+78
-57
lines changed

‎cli/cmd/delete.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,10 +46,11 @@ var deleteCmd = &cobra.Command{
4646
if len(args) == 1 {
4747
appName = args[0]
4848
} else {
49-
appName, err = appNameFromConfig()
49+
config, err := readConfig()
5050
if err != nil {
5151
errors.Exit(err)
5252
}
53+
appName = config.App.Name
5354
}
5455

5556
params := map[string]string{

‎cli/cmd/deploy.go‎

Lines changed: 32 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ import (
3232
"github.com/cortexlabs/cortex/pkg/operator/api/userconfig"
3333
)
3434

35+
var MaxProjectSize = 1024 * 1024 * 50
3536
var flagDeployForce bool
3637

3738
func init() {
@@ -51,7 +52,7 @@ var deployCmd = &cobra.Command{
5152

5253
func deploy(force bool, ignoreCache bool) {
5354
root := mustAppRoot()
54-
_, err := appNameFromConfig() // Check proper cortex.yaml
55+
config, err := readConfig() // Check proper cortex.yaml
5556
if err != nil {
5657
errors.Exit(err)
5758
}
@@ -66,34 +67,43 @@ func deploy(force bool, ignoreCache bool) {
6667
errors.Exit(errors.Wrap(err, "cortex.yaml", userconfig.ErrorReadConfig().Error()))
6768
}
6869

69-
projectPaths, err := files.ListDirRecursive(root, false,
70-
files.IgnoreCortexYAML,
71-
files.IgnoreHiddenFiles,
72-
files.IgnoreHiddenFolders,
73-
files.IgnorePythonGeneratedFiles,
74-
)
75-
if err != nil {
76-
errors.Exit(err)
70+
uploadBytes := map[string][]byte{
71+
"cortex.yaml": configBytes,
7772
}
7873

79-
projectZipBytes, err := zip.ToMem(&zip.Input{
80-
FileLists: []zip.FileListInput{
81-
{
82-
Sources: projectPaths,
83-
RemovePrefix: root,
74+
if config.AreProjectFilesRequired() {
75+
projectPaths, err := files.ListDirRecursive(root, false,
76+
files.IgnoreCortexYAML,
77+
files.IgnoreHiddenFiles,
78+
files.IgnoreHiddenFolders,
79+
files.IgnorePythonGeneratedFiles,
80+
)
81+
if err != nil {
82+
errors.Exit(err)
83+
}
84+
85+
projectZipBytes, err := zip.ToMem(&zip.Input{
86+
FileLists: []zip.FileListInput{
87+
{
88+
Sources: projectPaths,
89+
RemovePrefix: root,
90+
},
8491
},
85-
},
86-
})
92+
})
8793

88-
if err != nil {
89-
errors.Exit(errors.Wrap(err, "failed to zip project folder"))
94+
if err != nil {
95+
errors.Exit(errors.Wrap(err, "failed to zip project folder"))
96+
}
97+
98+
if len(projectZipBytes) > MaxProjectSize {
99+
errors.Exit(errors.New("zipped project folder exceeds " + s.Int(MaxProjectSize) + " bytes"))
100+
}
101+
102+
uploadBytes["project.zip"] = projectZipBytes
90103
}
91104

92105
uploadInput := &HTTPUploadInput{
93-
Bytes: map[string][]byte{
94-
"cortex.yaml": configBytes,
95-
"project.zip": projectZipBytes,
96-
},
106+
Bytes: uploadBytes,
97107
}
98108

99109
response, err := HTTPUpload("/deploy", uploadInput, params)

‎cli/cmd/lib_config_reader.go‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -66,22 +66,26 @@ func pythonPaths(dir string) []string {
6666
return pyPaths
6767
}
6868

69-
func appNameFromConfig() (string, error) {
69+
func readConfig() (*userconfig.Config, error) {
7070
appRoot := mustAppRoot()
71-
return userconfig.ReadAppName(filepath.Join(appRoot, "cortex.yaml"), "cortex.yaml")
71+
config, err := userconfig.ReadConfigFile(filepath.Join(appRoot, "cortex.yaml"), "cortex.yaml")
72+
if err != nil {
73+
return nil, err
74+
}
75+
return config, nil
7276
}
7377

7478
func AppNameFromFlagOrConfig() (string, error) {
7579
if flagAppName != "" {
7680
return flagAppName, nil
7781
}
7882

79-
appName, err := appNameFromConfig()
83+
config, err := readConfig()
8084
if err != nil {
8185
return "", err
8286
}
8387

84-
return appName, nil
88+
return config.App.Name, nil
8589
}
8690

8791
func IsAppNameSpecified() bool {

‎pkg/operator/api/userconfig/apis.go‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -300,6 +300,10 @@ func (api *API) Validate(projectFileMap map[string][]byte) error {
300300
return nil
301301
}
302302

303+
func (api *API) AreProjectFilesRequired() bool {
304+
return api.RequestHandler != nil
305+
}
306+
303307
func (api *API) GetResourceType() resource.Type {
304308
return resource.APIType
305309
}
@@ -311,3 +315,12 @@ func (apis APIs) Names() []string {
311315
}
312316
return names
313317
}
318+
319+
func (apis APIs) AreProjectFilesRequired() bool {
320+
for _, api := range apis {
321+
if api.AreProjectFilesRequired() {
322+
return true
323+
}
324+
}
325+
return false
326+
}

‎pkg/operator/api/userconfig/config.go‎

Lines changed: 18 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -37,15 +37,21 @@ var typeFieldValidation = &cr.StructFieldValidation{
3737
}
3838

3939
func (config *Config) Validate(projectBytes []byte) error {
40-
if err := config.App.Validate(); err != nil {
41-
return err
42-
}
40+
err := config.App.Validate()
4341

44-
projectFileMap, err := zip.UnzipMemToMem(projectBytes)
4542
if err != nil {
4643
return err
4744
}
4845

46+
projectFileMap := make(map[string][]byte)
47+
48+
if config.AreProjectFilesRequired() {
49+
projectFileMap, err = zip.UnzipMemToMem(projectBytes)
50+
if err != nil {
51+
return err
52+
}
53+
}
54+
4955
if config.APIs != nil {
5056
if err := config.APIs.Validate(projectFileMap); err != nil {
5157
return err
@@ -55,6 +61,10 @@ func (config *Config) Validate(projectBytes []byte) error {
5561
return nil
5662
}
5763

64+
func (config *Config) AreProjectFilesRequired() bool {
65+
return config.APIs.AreProjectFilesRequired()
66+
}
67+
5868
func New(filePath string, configBytes []byte) (*Config, error) {
5969
var err error
6070

@@ -118,16 +128,16 @@ func New(filePath string, configBytes []byte) (*Config, error) {
118128
return config, nil
119129
}
120130

121-
func ReadAppName(filePath string, relativePath string) (string, error) {
131+
func ReadConfigFile(filePath string, relativePath string) (*Config, error) {
122132
configBytes, err := files.ReadFileBytes(filePath)
123133
if err != nil {
124-
return "", errors.Wrap(err, relativePath, ErrorReadConfig().Error())
134+
return nil, errors.Wrap(err, relativePath, ErrorReadConfig().Error())
125135
}
126136

127137
config, err := New(relativePath, configBytes)
128138
if err != nil {
129-
return "", err
139+
return nil, err
130140
}
131141

132-
return config.App.Name, nil
142+
return config, nil
133143
}

‎pkg/operator/context/context.go‎

Lines changed: 5 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -69,14 +69,11 @@ func New(
6969
}
7070
ctx.APIs = apis
7171

72-
for _, api := range ctx.APIs {
73-
if api.RequestHandler != nil {
74-
ctx.ProjectID = projectID
75-
ctx.ProjectKey = filepath.Join(consts.ProjectsDir, ctx.ProjectID+".zip")
76-
if err = config.AWS.UploadBytesToS3(projectBytes, ctx.ProjectKey); err != nil {
77-
return nil, err
78-
}
79-
break
72+
if userconf.AreProjectFilesRequired() {
73+
ctx.ProjectID = projectID
74+
ctx.ProjectKey = filepath.Join(consts.ProjectsDir, ctx.ProjectID+".zip")
75+
if err = config.AWS.UploadBytesToS3(projectBytes, ctx.ProjectKey); err != nil {
76+
return nil, err
8077
}
8178
}
8279

‎pkg/operator/endpoints/deploy.go‎

Lines changed: 0 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -48,10 +48,6 @@ func Deploy(w http.ResponseWriter, r *http.Request) {
4848
}
4949

5050
projectBytes, err := files.ReadReqFile(r, "project.zip")
51-
if err != nil {
52-
RespondError(w, errors.WithStack(err))
53-
return
54-
}
5551

5652
userconf, err := userconfig.New("cortex.yaml", configBytes)
5753
if err != nil {
@@ -65,22 +61,12 @@ func Deploy(w http.ResponseWriter, r *http.Request) {
6561
return
6662
}
6763

68-
if len(projectBytes) == 0 {
69-
RespondError(w, ErrorFormFileMustBeProvided("project.zip"))
70-
return
71-
}
72-
7364
ctx, err := ocontext.New(userconf, projectBytes, ignoreCache)
7465
if err != nil {
7566
RespondError(w, err)
7667
return
7768
}
7869

79-
if err != nil {
80-
RespondError(w, err)
81-
return
82-
}
83-
8470
err = workloads.PopulateWorkloadIDs(ctx)
8571
if err != nil {
8672
RespondError(w, err)

0 commit comments

Comments
 (0)