Option to truncate appstructure (#485)

* Add option to truncate titles and descriptions in appstructure to a defined value to avoid unnecessary memory impact. * Workaround for engine ws bug
qlik-oss · Jun 24, 2024 · 41ebc32 · 41ebc32
1 parent e387164
commit 41ebc32
Show file tree

Hide file tree

Showing 6 changed files with 59 additions and 42 deletions.
diff --git a/config/appstructure.go b/config/appstructure.go
@@ -48,6 +48,7 @@ type (
 		logEntry      *logger.LogEntry
 		report        AppStructureReport
 		structureLock sync.Mutex
+		truncateTo    int
 	}
 
 	// Appstructure and warnings in consumable form
@@ -540,6 +541,7 @@ func (structure *GeneratedAppStructure) handleObject(typ string, obj *appstructu
 	metaDef := helpers.NewDataPath("/qMetaDef")
 	rawMetaDef, _ := metaDef.Lookup(properties)
 	_ = json.Unmarshal(rawMetaDef, &obj.MetaDef)
+	truncateMeta(&obj.MetaDef, structure.truncateTo)
 
 	enumTyp, _ := appstructure.ObjectTypeEnumMap.Int(typ) // 0 will be default in case of "error" == ObjectTypeDefault
 
@@ -679,10 +681,8 @@ func (structure *GeneratedAppStructure) handleMeasure(ctx context.Context, app *
 	rawMeasure, err := measurePath.Lookup(obj.RawBaseProperties)
 	if err != nil {
 		structure.warn(id, typ, "measure", AppStructureWarningMeasureNotFound, "measure definition not found")
-	} else {
-		if err := json.Unmarshal(rawMeasure, &measure); err != nil {
-			return errors.WithStack(err)
-		}
+	} else if err := json.Unmarshal(rawMeasure, &measure); err != nil {
+		return errors.WithStack(err)
 	}
 
 	// Save meta information to structure
@@ -691,11 +691,10 @@ func (structure *GeneratedAppStructure) handleMeasure(ctx context.Context, app *
 	rawMeta, err := metaPath.Lookup(obj.RawBaseProperties)
 	if err != nil {
 		structure.warn(id, typ, "measure", AppStructureWarningMetaInformationNotFound, "measure has not meta information")
-	} else {
-		if err := json.Unmarshal(rawMeta, &meta); err != nil {
-			return errors.WithStack(err)
-		}
+	} else if err := json.Unmarshal(rawMeta, &meta); err != nil {
+		return errors.WithStack(err)
 	}
+	truncateMeta(&meta, structure.truncateTo)
 
 	obj.Measures = []appstructure.AppStructureMeasureMeta{
 		{
@@ -724,10 +723,8 @@ func (structure *GeneratedAppStructure) handleDimension(ctx context.Context, app
 	rawDimension, err := dimensionPath.Lookup(obj.RawBaseProperties)
 	if err != nil {
 		structure.warn(id, typ, "dimension", AppStructureWarningDimensionNotFound, "dimension defintion not found")
-	} else {
-		if err := json.Unmarshal(rawDimension, &dimension); err != nil {
-			return errors.WithStack(err)
-		}
+	} else if err := json.Unmarshal(rawDimension, &dimension); err != nil {
+		return errors.WithStack(err)
 	}
 
 	// Add dimension meta information to structure
@@ -736,11 +733,10 @@ func (structure *GeneratedAppStructure) handleDimension(ctx context.Context, app
 	rawMeta, err := metaPath.Lookup(obj.RawBaseProperties)
 	if err != nil {
 		structure.warn(id, typ, "dimension", AppStructureWarningMetaInformationNotFound, "dimension has not meta information")
-	} else {
-		if err := json.Unmarshal(rawMeta, &meta); err != nil {
-			return errors.WithStack(err)
-		}
+	} else if err := json.Unmarshal(rawMeta, &meta); err != nil {
+		return errors.WithStack(err)
 	}
+	truncateMeta(&meta, structure.truncateTo)
 
 	obj.Dimensions = []appstructure.AppStructureDimensionMeta{
 		{
@@ -847,11 +843,10 @@ func (structure *GeneratedAppStructure) handleBookmark(ctx context.Context, app
 	rawMeta, err := metaPath.Lookup(properties)
 	if err != nil {
 		structure.warn(id, typ, "", AppStructureWarningMetaInformationNotFound, "bookmark has no meta information")
-	} else {
-		if err = json.Unmarshal(rawMeta, &meta); err != nil {
-			structure.warn(id, typ, "", AppStructureWarningMetaInformationNotFound, fmt.Sprintf("bookmark failed to unmarshal meta information: %s", err))
-		}
+	} else if err = json.Unmarshal(rawMeta, &meta); err != nil {
+		structure.warn(id, typ, "", AppStructureWarningMetaInformationNotFound, fmt.Sprintf("bookmark failed to unmarshal meta information: %s", err))
 	}
+	truncateMeta(&meta, structure.truncateTo)
 
 	structureBookmark.Title = meta.Title
 	structureBookmark.Description = meta.Description
@@ -987,3 +982,11 @@ func (report *AppStructureReport) AddWarning(warning AppStructureWarning) {
 
 	report.warnings = append(report.warnings, warning)
 }
+
+func truncateMeta(meta *appstructure.MetaDef, truncateTo int) {
+	if truncateTo < 1 {
+		return
+	}
+	meta.Title = meta.Title[:helpers.Min(len(meta.Title), truncateTo)]
+	meta.Description = meta.Description[:helpers.Min(len(meta.Description), truncateTo)]
+}
diff --git a/config/getappstructure.go b/config/getappstructure.go
@@ -14,8 +14,10 @@ import (
 
 type (
 	GetAppStructureSettings struct {
-		IncludeRaw    bool `json:"includeRaw,omitempty"`
-		AppStructures map[string]*GeneratedAppStructure
+		IncludeRaw bool `json:"includeRaw,omitempty"`
+		// TruncateStringsTo truncates non significant strings to size if set to > 0
+		TruncateStringsTo int `json:"truncate,omitempty"`
+		AppStructures     map[string]*GeneratedAppStructure
 	}
 )
 
@@ -59,6 +61,7 @@ func (settings *GetAppStructureSettings) Execute(sessionState *session.State, ac
 		sessionState.LogEntry,
 		AppStructureReport{},
 		sync.Mutex{},
+		settings.TruncateStringsTo,
 	}
 	structure.logEntry = sessionState.LogEntry
 

diff --git a/go.mod b/go.mod
@@ -22,6 +22,7 @@ require (
 	github.com/shiena/ansicolor v0.0.0-20230509054315-a9deabde6e02
 	github.com/spf13/cobra v1.8.1
 	github.com/stretchr/testify v1.9.0
+	golang.org/x/exp v0.0.0-20240604190554-fc45aab8b7f8
 )
 
 require (
@@ -44,7 +45,7 @@ require (
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/tus/tusd v1.3.0 // indirect
 	golang.org/x/net v0.25.0 // indirect
-	golang.org/x/sync v0.3.0 // indirect
+	golang.org/x/sync v0.7.0 // indirect
 	golang.org/x/sys v0.20.0 // indirect
 	google.golang.org/protobuf v1.33.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect

diff --git a/go.sum b/go.sum
@@ -43,12 +43,8 @@ github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU
 github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM=
 github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og=
 github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=
-github.com/gobwas/ws v1.3.2 h1:zlnbNHxumkRvfPWgfXu8RBwyNR1x8wh9cf5PTOCqs9Q=
-github.com/gobwas/ws v1.3.2/go.mod h1:hRKAFb8wOxFROYNsT1bqfWnhX+b5MFeJM9r2ZSwg/KY=
 github.com/gobwas/ws v1.4.0 h1:CTaoG1tojrh4ucGPcoJFiAQUAsEWekEWvLy7GsVNqGs=
 github.com/gobwas/ws v1.4.0/go.mod h1:G3gNqMNtPppf5XUz7O4shetPpcZ1VJ7zt18dlUeakrc=
-github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
-github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
 github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA=
 github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
@@ -85,8 +81,6 @@ github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY
 github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
 github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
-github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek=
-github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
 github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKeRZfjY=
 github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
@@ -133,8 +127,6 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
 github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
-github.com/prometheus/client_golang v1.19.0 h1:ygXvpU1AoN1MhdzckN+PyD9QJOSD4x7kmXYlnfbA6JU=
-github.com/prometheus/client_golang v1.19.0/go.mod h1:ZRM9uEAypZakd+q/x7+gmsvXdURP+DABIEIjnmDdp+k=
 github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE=
 github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
@@ -149,17 +141,13 @@ github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R
 github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo=
 github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo=
-github.com/qlik-oss/enigma-go/v4 v4.1.0 h1:LlgB0SyW+YDzhScbt7XV6/8JOwfRtG5MIi+TPzcbxXk=
-github.com/qlik-oss/enigma-go/v4 v4.1.0/go.mod h1:2JAdz/EGlEek4OnyY26s7v+Wp4pw0QX9WJrWOXIEPJc=
 github.com/qlik-oss/enigma-go/v4 v4.2.0 h1:LUNQvRIOWjlBV56IK85QcJbjdkkgShfnK3bQX/qnnao=
 github.com/qlik-oss/enigma-go/v4 v4.2.0/go.mod h1:XE2m2KWbdrhMJxVskR4vIHTZlDcP2Sh88lbkd3f//pA=
 github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
 github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
 github.com/rs/dnscache v0.0.0-20230804202142-fc85eb664529 h1:18kd+8ZUlt/ARXhljq+14TwAoKa61q6dX8jtwOf6DH8=
 github.com/rs/dnscache v0.0.0-20230804202142-fc85eb664529/go.mod h1:qe5TWALJ8/a1Lqznoc5BDHpYX/8HU60Hm2AwRmqzxqA=
 github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
-github.com/rs/zerolog v1.32.0 h1:keLypqrlIjaFsbmJOBdB/qvyF8KEtCWHwobLp5l/mQ0=
-github.com/rs/zerolog v1.32.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss=
 github.com/rs/zerolog v1.33.0 h1:1cU2KZkvPxNyfgEmhHAz/1A9Bz+llsdYzklWFzgp0r8=
 github.com/rs/zerolog v1.33.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
@@ -196,6 +184,8 @@ go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20240604190554-fc45aab8b7f8 h1:LoYXNGAShUG3m/ehNk4iFctuhGX/+R1ZpfJ4/ia80JM=
+golang.org/x/exp v0.0.0-20240604190554-fc45aab8b7f8/go.mod h1:jj3sYF3dwk5D+ghuXyeI3r5MFf+NT2An6/9dOA95KSI=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
 golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@@ -209,8 +199,6 @@ golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73r
 golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs=
-golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
 golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac=
 golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -221,8 +209,8 @@ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E=
-golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
+golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
+golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -233,8 +221,6 @@ golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
-golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
 golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

diff --git a/helpers/math.go b/helpers/math.go
@@ -0,0 +1,19 @@
+package helpers
+
+import "golang.org/x/exp/constraints"
+
+// Min returns the least of a and b
+func Min[T constraints.Ordered](a, b T) T {
+	if a < b {
+		return a
+	}
+	return b
+}
+
+// Max returns the greatest of a and b
+func Max[T constraints.Ordered](a, b T) T {
+	if a > b {
+		return a
+	}
+	return b
+}
diff --git a/wsdialer/wsdialer.go b/wsdialer/wsdialer.go
@@ -22,6 +22,10 @@ import (
 	"github.com/qlik-oss/gopherciser/helpers"
 )
 
+var (
+	ErrFrameTooLarge = wsutil.ErrFrameTooLarge
+)
+
 type (
 	// ReConnectSettings settings for automatically reconnecting websocket
 	ReConnectSettings struct {
@@ -194,7 +198,8 @@ func readMessage(r io.Reader, m []wsutil.Message, maxFrameSize int64) ([]wsutil.
 			m = append(m, wsutil.Message{OpCode: hdr.OpCode, Payload: bts})
 			return nil
 		},
-		MaxFrameSize: maxFrameSize,
+		MaxFrameSize:    maxFrameSize,
+		SkipHeaderCheck: true,
 	}
 	h, err := rd.NextFrame()
 	if err != nil {