From 17ff800df8123e828a6bba8ab2a2eef6929a68b6 Mon Sep 17 00:00:00 2001 From: zekiahmetbayar Date: Mon, 8 Jan 2024 17:39:11 +0300 Subject: [PATCH 1/4] feature: Alternative Packages --- .github/workflows/build.yml | 2 ++ app/controllers/packages/packages.go | 9 +++--- app/entities/AlternativePackage.go | 9 ++++++ app/entities/Package.go | 12 ++++--- cmd/server/main.go | 9 +++++- internal/constants/constants.go | 5 +-- internal/migrations/migrate.go | 3 ++ internal/seeds/seeds.go | 47 ++++++++++++++++++++++++++++ pkg/discovery/discovery.go | 13 ++++++++ storage/alternatives.csv | 2 ++ 10 files changed, 100 insertions(+), 11 deletions(-) create mode 100644 app/entities/AlternativePackage.go create mode 100644 internal/seeds/seeds.go create mode 100644 storage/alternatives.csv diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2603418..d8e9a87 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -71,7 +71,9 @@ jobs: mkdir -p $SERVER_BIN_PATH mkdir -p $SERVER_BIN_PATH/reports mkdir -p $SERVER_BIN_PATH/wmi + mkdir -p $SERVER_BIN_PATH/storage mv wmi.so $SERVER_BIN_PATH/wmi/ + mv storage/alternatives.csv $SERVER_BIN_PATH/storage/ cp ./scripts/debian/control $DEBIAN_PATH/control sed -i s/%VERSION%/${{ github.run_number }}/g $DEBIAN_PATH/control diff --git a/app/controllers/packages/packages.go b/app/controllers/packages/packages.go index b321367..fa2bb1e 100644 --- a/app/controllers/packages/packages.go +++ b/app/controllers/packages/packages.go @@ -8,6 +8,7 @@ import ( "github.com/limanmys/inventory-server/internal/search" "github.com/limanmys/inventory-server/pkg/jobs" "github.com/limanmys/inventory-server/pkg/reporter" + "gorm.io/gorm/clause" ) // Index, returns asset's packages @@ -15,12 +16,12 @@ func Index(c *fiber.Ctx) error { // Build sql query sub_query := database.Connection(). Model(&entities.Package{}). - Select("packages.name", "count(*)", "null as updated_at", "null as deleted_at"). + Select("packages.name", "count(*)", "null as updated_at", "null as deleted_at", "alternative_package_id"). Joins("inner join asset_packages ap on ap.package_id = packages.id"). Joins("inner join assets on assets.id = ap.asset_id"). - Group("packages.name").Order("count desc") + Group("packages.name").Group("alternative_package_id").Order("count desc") - db := database.Connection().Table("(?) as t1", sub_query) + db := database.Connection().Preload(clause.Associations).Table("(?) as t1", sub_query) // Apply search, if exists if c.Query("search") != "" { @@ -28,7 +29,7 @@ func Index(c *fiber.Ctx) error { } // Get data - var packages []map[string]interface{} + var packages []entities.Package page, err := paginator.New(db, c).Paginate(&packages) if err != nil { return err diff --git a/app/entities/AlternativePackage.go b/app/entities/AlternativePackage.go new file mode 100644 index 0000000..160cbe4 --- /dev/null +++ b/app/entities/AlternativePackage.go @@ -0,0 +1,9 @@ +package entities + +type AlternativePackage struct { + Base + Name string `json:"name"` + URL string `json:"url"` + PackageName string `json:"package_name"` + Packages []*Package `json:"packages"` +} diff --git a/app/entities/Package.go b/app/entities/Package.go index 42113c0..58b8df4 100644 --- a/app/entities/Package.go +++ b/app/entities/Package.go @@ -1,9 +1,13 @@ package entities +import "github.com/google/uuid" + type Package struct { Base - Name string `json:"name"` - Version string `json:"version"` - Vendor string `json:"vendor"` - Assets []*Asset `json:"assets" gorm:"many2many:asset_packages"` + Name string `json:"name"` + Version string `json:"version"` + Vendor string `json:"vendor"` + Assets []*Asset `json:"assets" gorm:"many2many:asset_packages"` + AlternativePackageID *uuid.UUID `json:"alternative_package_id"` + AlternativePackage *AlternativePackage `json:"alternative_package"` } diff --git a/cmd/server/main.go b/cmd/server/main.go index d0243d1..1d662d5 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -12,13 +12,20 @@ import ( "github.com/gofiber/fiber/v2/middleware/recover" "github.com/limanmys/inventory-server/app/routes" "github.com/limanmys/inventory-server/internal/migrations" + "github.com/limanmys/inventory-server/internal/seeds" "github.com/limanmys/inventory-server/internal/server" ) func main() { // Migrate tables if !fiber.IsChild() { - migrations.Migrate() + //Migrate tables + if err := migrations.Migrate(); err != nil { + log.Println("error when migrating tables, ", err.Error()) + } + + // Seed alternative packages + seeds.Init() } // Create Fiber App diff --git a/internal/constants/constants.go b/internal/constants/constants.go index a437490..7478384 100644 --- a/internal/constants/constants.go +++ b/internal/constants/constants.go @@ -1,6 +1,7 @@ package constants const ( - REPORT_SAVE_PATH = "./reports/" - WMI_SO_PATH = "./wmi/wmi.so" + REPORT_SAVE_PATH = "./reports/" + WMI_SO_PATH = "./wmi/wmi.so" + ALTERNATIVES_CSV_PATH = "./storage/alternatives.csv" ) diff --git a/internal/migrations/migrate.go b/internal/migrations/migrate.go index e0aee8d..f54b96b 100644 --- a/internal/migrations/migrate.go +++ b/internal/migrations/migrate.go @@ -6,6 +6,9 @@ import ( ) func Migrate() error { + if err := database.Connection().AutoMigrate(&entities.AlternativePackage{}); err != nil { + return err + } if err := database.Connection().AutoMigrate(&entities.Discovery{}); err != nil { return err } diff --git a/internal/seeds/seeds.go b/internal/seeds/seeds.go new file mode 100644 index 0000000..9c4614f --- /dev/null +++ b/internal/seeds/seeds.go @@ -0,0 +1,47 @@ +package seeds + +import ( + "encoding/csv" + "log" + "os" + + "github.com/limanmys/inventory-server/app/entities" + "github.com/limanmys/inventory-server/internal/constants" + "github.com/limanmys/inventory-server/internal/database" +) + +// Init, seeds alternative packages +func Init() { + // Open alternatives.csv + file, err := os.Open(constants.ALTERNATIVES_CSV_PATH) + if err != nil { + log.Println("error when reading package alternatives, " + err.Error()) + return + } + + // Close file after function end + defer file.Close() + + // Read all rows from csv + records, err := csv.NewReader(file).ReadAll() + if err != nil { + log.Println("error when reading records, " + err.Error()) + return + } + + // Create records + for idx, record := range records { + // If row is wrong or title row + if len(record) != 3 || idx == 0 { + continue + } + // Create record on database + if err := database.Connection().Where("name = ?", record[0]).FirstOrCreate(&entities.AlternativePackage{ + Name: record[0], + URL: record[1], + PackageName: record[2], + }).Error; err != nil { + log.Println("error when creating alternative package, " + err.Error()) + } + } +} diff --git a/pkg/discovery/discovery.go b/pkg/discovery/discovery.go index 33644bf..d9ead59 100644 --- a/pkg/discovery/discovery.go +++ b/pkg/discovery/discovery.go @@ -4,6 +4,7 @@ import "C" import ( "encoding/base64" "encoding/json" + "fmt" "github.com/limanmys/inventory-server/app/entities" "github.com/limanmys/inventory-server/internal/constants" @@ -125,12 +126,24 @@ func Start(discovery entities.Discovery) { // If package does not exists if count == 0 { + + // Check package has alternative + var alternative entities.AlternativePackage + database.Connection().Model(&entities.AlternativePackage{}). + Where("LOWER(package_name) LIKE LOWER(?)", fmt.Sprintf("%%%s%%", pkg.Name)). + First(&alternative) + + if alternative.ID != nil { + pkg.AlternativePackageID = alternative.ID + } + database.Connection().Clauses(clause.Returning{}).Create(&pkg) } else { // Find package database.Connection().Model(&entities.Package{}). Where("name = ? and version = ?", pkg.Name, pkg.Version).First(&pkg) } + // Append package to asset packages assetPackages = append(assetPackages, pkg) } diff --git a/storage/alternatives.csv b/storage/alternatives.csv new file mode 100644 index 0000000..0ea806a --- /dev/null +++ b/storage/alternatives.csv @@ -0,0 +1,2 @@ +Name,URL,Package Name +Okular,https://okular.kde.org/tr/,Adobe Acrobat (64-bit) From c41f8b9076129efcb599bbc6ed3f4b9fd9fc176c Mon Sep 17 00:00:00 2001 From: zekiahmetbayar Date: Tue, 9 Jan 2024 13:05:01 +0300 Subject: [PATCH 2/4] fix: Asset count on package index issue --- app/controllers/packages/packages.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/app/controllers/packages/packages.go b/app/controllers/packages/packages.go index fa2bb1e..12689e3 100644 --- a/app/controllers/packages/packages.go +++ b/app/controllers/packages/packages.go @@ -11,6 +11,11 @@ import ( "gorm.io/gorm/clause" ) +type PackageWithAssetCount struct { + entities.Package + Count int `json:"count"` +} + // Index, returns asset's packages func Index(c *fiber.Ctx) error { // Build sql query @@ -29,7 +34,7 @@ func Index(c *fiber.Ctx) error { } // Get data - var packages []entities.Package + var packages []PackageWithAssetCount page, err := paginator.New(db, c).Paginate(&packages) if err != nil { return err From ce4ecf8f66d19a88244db9a955fe9cf14c0d00d1 Mon Sep 17 00:00:00 2001 From: zekiahmetbayar Date: Thu, 11 Jan 2024 16:56:50 +0300 Subject: [PATCH 3/4] feature: Added alternative package to package report --- app/controllers/packages/packages.go | 10 +++++++--- pkg/reporter/reporter.go | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/app/controllers/packages/packages.go b/app/controllers/packages/packages.go index 12689e3..bb48024 100644 --- a/app/controllers/packages/packages.go +++ b/app/controllers/packages/packages.go @@ -53,13 +53,17 @@ func Report(c *fiber.Ctx) error { // Build query db := database.Connection(). Model(&entities.Package{}). - Select("packages.name", "count(*)"). + Select( + "packages.name", + "count(*)", + "coalesce(alternative_packages.name, '-') as alternative_package"). Joins("inner join asset_packages ap on ap.package_id = packages.id"). Joins("inner join assets on assets.id = ap.asset_id"). - Group("packages.name").Order("count desc") + Joins("left join alternative_packages on alternative_packages.id = packages.alternative_package_id "). + Group("packages.name, alternative_packages.name").Order("count desc") // Create report as go routine - go reporter.CreatePackageReport(job, db, []string{"name", "count"}) + go reporter.CreatePackageReport(job, db, []string{"name", "count", "alternative_package"}) return c.JSON(fiber.Map{"id": job.ID.String()}) } diff --git a/pkg/reporter/reporter.go b/pkg/reporter/reporter.go index cfc6acf..6f6b5ce 100644 --- a/pkg/reporter/reporter.go +++ b/pkg/reporter/reporter.go @@ -25,7 +25,7 @@ func CreatePackageReport(job *entities.Job, db *gorm.DB, columns []string) { // Build report body body := map[string]interface{}{ "date": time.Now().Format("01-02-2006 15:04:05"), - "header": "Inventory Server | Packages Report", + "header": "Packages Report", "readable_columns": humanize(columns), "template_id": "template.docx", "columns": columns, From 14829b9489eeb58a887f87a623ec75f9ad331401 Mon Sep 17 00:00:00 2001 From: zekiahmetbayar Date: Fri, 12 Jan 2024 13:56:59 +0300 Subject: [PATCH 4/4] feature: Test run parameter --- cmd/server/main.go | 13 +++++++++++-- scripts/debian/postinst | 10 +++++----- scripts/redhat/inventory-server.spec | 10 +++++----- 3 files changed, 21 insertions(+), 12 deletions(-) diff --git a/cmd/server/main.go b/cmd/server/main.go index 1d662d5..25895e6 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -1,6 +1,7 @@ package main import ( + "flag" "log" _ "github.com/joho/godotenv/autoload" @@ -17,6 +18,9 @@ import ( ) func main() { + runType := flag.String("type", "admin", "Server's run type.") + flag.Parse() + // Migrate tables if !fiber.IsChild() { //Migrate tables @@ -46,6 +50,11 @@ func main() { // Mount routes routes.Routes(app) - // Start server - log.Fatal(app.Listen("127.0.0.1:7806")) + if *runType == "admin" { + // Start server + log.Fatal(app.Listen("127.0.0.1:7806")) + } else if *runType == "test" { + // Start test server + log.Fatal(app.Listen("0.0.0.0:7825")) + } } diff --git a/scripts/debian/postinst b/scripts/debian/postinst index cfd04d3..f8b5344 100644 --- a/scripts/debian/postinst +++ b/scripts/debian/postinst @@ -7,11 +7,11 @@ fi echo """ [Unit] -Description=Inventory Server +Description=Inventory Server (%I) [Service] Type=simple WorkingDirectory=/opt/inventory-server -ExecStart=/opt/inventory-server/inventory-server +ExecStart=/opt/inventory-server/inventory-server -type=%i Restart=always RestartSec=10 SyslogIdentifier=inventory @@ -20,8 +20,8 @@ User=root Group=root [Install] WantedBy=multi-user.target - """ > /etc/systemd/system/inventory-server.service + """ > /etc/systemd/system/inventory-server@.service systemctl daemon-reload -systemctl enable inventory-server.service -systemctl restart inventory-server.service \ No newline at end of file +systemctl enable inventory-server@admin.service +systemctl restart inventory-server@admin.service \ No newline at end of file diff --git a/scripts/redhat/inventory-server.spec b/scripts/redhat/inventory-server.spec index 3565bc6..5fd025c 100644 --- a/scripts/redhat/inventory-server.spec +++ b/scripts/redhat/inventory-server.spec @@ -29,11 +29,11 @@ fi echo """ [Unit] -Description=Inventory Server +Description=Inventory Server (%I) [Service] Type=simple WorkingDirectory=/opt/inventory-server -ExecStart=/opt/inventory-server/inventory-server +ExecStart=/opt/inventory-server/inventory-server -type=%i Restart=always RestartSec=10 SyslogIdentifier=inventory @@ -42,11 +42,11 @@ User=root Group=root [Install] WantedBy=multi-user.target - """ > /etc/systemd/system/inventory-server.service + """ > /etc/systemd/system/inventory-server@.service systemctl daemon-reload -systemctl enable inventory-server.service -systemctl restart inventory-server.service +systemctl enable inventory-server@admin.service +systemctl restart inventory-server@admin.service %clean