Skip to content

Commit

Permalink
Merge remote-tracking branch 'upstream/dev'
Browse files Browse the repository at this point in the history
  • Loading branch information
@yahavi
yahavi committed Nov 30, 2023
2 parents 82fecb0 + 48419a7 commit d4e93a4
Show file tree
Hide file tree
Showing 66 changed files with 149,445 additions and 915 deletions.
40 changes: 40 additions & 0 deletions artifactory/commands/transferfiles/filediff_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,3 +32,43 @@ func TestConvertResultsToFileRepresentation(t *testing.T) {
assert.Equal(t, []api.FileRepresentation{testCase.expectedOutput}, files)
}
}

var generateDiffAqlQueryTestCases = []struct {
paginationOffset int
disabledDistinctiveAql bool
expectedAql string
}{
{0, false, "items.find({\"$and\":[{\"modified\":{\"$gte\":\"1\"}},{\"modified\":{\"$lt\":\"2\"}},{\"repo\":\"repo1\",\"type\":\"any\"}]}).include(\"repo\",\"path\",\"name\",\"type\",\"modified\",\"size\").sort({\"$asc\":[\"name\",\"path\"]}).offset(0).limit(10000)"},
{0, true, "items.find({\"$and\":[{\"modified\":{\"$gte\":\"1\"}},{\"modified\":{\"$lt\":\"2\"}},{\"repo\":\"repo1\",\"type\":\"any\"}]}).include(\"repo\",\"path\",\"name\",\"type\",\"modified\",\"size\").sort({\"$asc\":[\"name\",\"path\"]}).offset(0).limit(10000).distinct(false)"},
{2, false, "items.find({\"$and\":[{\"modified\":{\"$gte\":\"1\"}},{\"modified\":{\"$lt\":\"2\"}},{\"repo\":\"repo1\",\"type\":\"any\"}]}).include(\"repo\",\"path\",\"name\",\"type\",\"modified\",\"size\").sort({\"$asc\":[\"name\",\"path\"]}).offset(20000).limit(10000)"},
{2, true, "items.find({\"$and\":[{\"modified\":{\"$gte\":\"1\"}},{\"modified\":{\"$lt\":\"2\"}},{\"repo\":\"repo1\",\"type\":\"any\"}]}).include(\"repo\",\"path\",\"name\",\"type\",\"modified\",\"size\").sort({\"$asc\":[\"name\",\"path\"]}).offset(20000).limit(10000).distinct(false)"},
}

func TestGenerateDiffAqlQuery(t *testing.T) {
for _, testCase := range generateDiffAqlQueryTestCases {
t.Run("", func(*testing.T) {
results := generateDiffAqlQuery(repo1Key, "1", "2", testCase.paginationOffset, testCase.disabledDistinctiveAql)
assert.Equal(t, testCase.expectedAql, results)
})
}
}

var generateDockerManifestAqlQueryTestCases = []struct {
paginationOffset int
disabledDistinctiveAql bool
expectedAql string
}{
{0, false, "items.find({\"$and\":[{\"repo\":\"repo1\"},{\"modified\":{\"$gte\":\"1\"}},{\"modified\":{\"$lt\":\"2\"}},{\"$or\":[{\"name\":\"manifest.json\"},{\"name\":\"list.manifest.json\"}]}]}).include(\"repo\",\"path\",\"name\",\"type\",\"modified\").sort({\"$asc\":[\"name\",\"path\"]}).offset(0).limit(10000)"},
{0, true, "items.find({\"$and\":[{\"repo\":\"repo1\"},{\"modified\":{\"$gte\":\"1\"}},{\"modified\":{\"$lt\":\"2\"}},{\"$or\":[{\"name\":\"manifest.json\"},{\"name\":\"list.manifest.json\"}]}]}).include(\"repo\",\"path\",\"name\",\"type\",\"modified\").sort({\"$asc\":[\"name\",\"path\"]}).offset(0).limit(10000).distinct(false)"},
{2, false, "items.find({\"$and\":[{\"repo\":\"repo1\"},{\"modified\":{\"$gte\":\"1\"}},{\"modified\":{\"$lt\":\"2\"}},{\"$or\":[{\"name\":\"manifest.json\"},{\"name\":\"list.manifest.json\"}]}]}).include(\"repo\",\"path\",\"name\",\"type\",\"modified\").sort({\"$asc\":[\"name\",\"path\"]}).offset(20000).limit(10000)"},
{2, true, "items.find({\"$and\":[{\"repo\":\"repo1\"},{\"modified\":{\"$gte\":\"1\"}},{\"modified\":{\"$lt\":\"2\"}},{\"$or\":[{\"name\":\"manifest.json\"},{\"name\":\"list.manifest.json\"}]}]}).include(\"repo\",\"path\",\"name\",\"type\",\"modified\").sort({\"$asc\":[\"name\",\"path\"]}).offset(20000).limit(10000).distinct(false)"},
}

func TestGenerateDockerManifestAqlQuery(t *testing.T) {
for _, testCase := range generateDockerManifestAqlQueryTestCases {
t.Run("", func(*testing.T) {
results := generateDockerManifestAqlQuery(repo1Key, "1", "2", testCase.paginationOffset, testCase.disabledDistinctiveAql)
assert.Equal(t, testCase.expectedAql, results)
})
}
}
20 changes: 12 additions & 8 deletions artifactory/commands/transferfiles/filesdiff.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -214,7 +214,7 @@ func (f *filesDiffPhase) getTimeFrameFilesDiff(fromTimestamp, toTimestamp string
}

func (f *filesDiffPhase) getNonDockerTimeFrameFilesDiff(fromTimestamp, toTimestamp string, paginationOffset int) (aqlResult *servicesUtils.AqlSearchResult, err error) {
query := generateDiffAqlQuery(f.repoKey, fromTimestamp, toTimestamp, paginationOffset)
query := generateDiffAqlQuery(f.repoKey, fromTimestamp, toTimestamp, paginationOffset, f.disabledDistinctiveAql)
return runAql(f.context, f.srcRtDetails, query)
}

Expand All @@ -225,7 +225,7 @@ func (f *filesDiffPhase) getNonDockerTimeFrameFilesDiff(fromTimestamp, toTimesta
// to get all artifacts in its path (that includes the "manifest.json" file itself and all its layouts).
func (f *filesDiffPhase) getDockerTimeFrameFilesDiff(fromTimestamp, toTimestamp string, paginationOffset int) (aqlResult *servicesUtils.AqlSearchResult, err error) {
// Get all newly created or modified manifest files ("manifest.json" and "list.manifest.json" files)
query := generateDockerManifestAqlQuery(f.repoKey, fromTimestamp, toTimestamp, paginationOffset)
query := generateDockerManifestAqlQuery(f.repoKey, fromTimestamp, toTimestamp, paginationOffset, f.disabledDistinctiveAql)
manifestFilesResult, err := runAql(f.context, f.srcRtDetails, query)
if err != nil {
return
Expand Down Expand Up @@ -261,11 +261,10 @@ func (f *filesDiffPhase) getDockerTimeFrameFilesDiff(fromTimestamp, toTimestamp
return
}

func generateDiffAqlQuery(repoKey, fromTimestamp, toTimestamp string, paginationOffset int) string {
func generateDiffAqlQuery(repoKey, fromTimestamp, toTimestamp string, paginationOffset int, disabledDistinctiveAql bool) string {
query := fmt.Sprintf(`items.find({"$and":[{"modified":{"$gte":"%s"}},{"modified":{"$lt":"%s"}},{"repo":"%s","type":"any"}]})`, fromTimestamp, toTimestamp, repoKey)
query += `.include("repo","path","name","type","modified","size")`
query += fmt.Sprintf(`.sort({"$asc":["modified"]}).offset(%d).limit(%d)`, paginationOffset*AqlPaginationLimit, AqlPaginationLimit)
return query
return query + generateAqlSortingPart(paginationOffset, disabledDistinctiveAql)
}

// This function generates an AQL that searches for all the content in the list of provided Artifactory paths.
Expand All @@ -283,10 +282,15 @@ func generateGetDirContentAqlQuery(repoKey string, paths []string) string {
}

// This function generates an AQL that searches for all files named "manifest.json" and "list.manifest.json" in a specific repository.
func generateDockerManifestAqlQuery(repoKey, fromTimestamp, toTimestamp string, paginationOffset int) string {
func generateDockerManifestAqlQuery(repoKey, fromTimestamp, toTimestamp string, paginationOffset int, disabledDistinctiveAql bool) string {
query := `items.find({"$and":`
query += fmt.Sprintf(`[{"repo":"%s"},{"modified":{"$gte":"%s"}},{"modified":{"$lt":"%s"}},{"$or":[{"name":"manifest.json"},{"name":"list.manifest.json"}]}`, repoKey, fromTimestamp, toTimestamp)
query += `]}).include("repo","path","name","type","modified")`
query += fmt.Sprintf(`.sort({"$asc":["modified"]}).offset(%d).limit(%d)`, paginationOffset*AqlPaginationLimit, AqlPaginationLimit)
return query
return query + generateAqlSortingPart(paginationOffset, disabledDistinctiveAql)
}

func generateAqlSortingPart(paginationOffset int, disabledDistinctiveAql bool) string {
sortingPart := fmt.Sprintf(`.sort({"$asc":["name","path"]}).offset(%d).limit(%d)`, paginationOffset*AqlPaginationLimit, AqlPaginationLimit)
sortingPart += appendDistinctIfNeeded(disabledDistinctiveAql)
return sortingPart
}
5 changes: 3 additions & 2 deletions artifactory/commands/transferfiles/fulltransfer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -273,7 +273,7 @@ func getFolderRelativePath(folderName, relativeLocation string) string {
}

func (m *fullTransferPhase) getDirectoryContentAql(relativePath string, paginationOffset int) (result []servicesUtils.ResultItem, lastPage bool, err error) {
query := generateFolderContentAqlQuery(m.repoKey, relativePath, paginationOffset)
query := generateFolderContentAqlQuery(m.repoKey, relativePath, paginationOffset, m.disabledDistinctiveAql)
aqlResults, err := runAql(m.context, m.srcRtDetails, query)
if err != nil {
return []servicesUtils.ResultItem{}, false, err
Expand All @@ -284,10 +284,11 @@ func (m *fullTransferPhase) getDirectoryContentAql(relativePath string, paginati
return
}

func generateFolderContentAqlQuery(repoKey, relativePath string, paginationOffset int) string {
func generateFolderContentAqlQuery(repoKey, relativePath string, paginationOffset int, disabledDistinctiveAql bool) string {
query := fmt.Sprintf(`items.find({"type":"any","$or":[{"$and":[{"repo":"%s","path":{"$match":"%s"},"name":{"$match":"*"}}]}]})`, repoKey, relativePath)
query += `.include("repo","path","name","type","size")`
query += fmt.Sprintf(`.sort({"$asc":["name"]}).offset(%d).limit(%d)`, paginationOffset*AqlPaginationLimit, AqlPaginationLimit)
query += appendDistinctIfNeeded(disabledDistinctiveAql)
return query
}

Expand Down
39 changes: 22 additions & 17 deletions artifactory/commands/transferfiles/longpropertycheck.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
"github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/transferfiles/api"
cmdutils "github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/utils"
"github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/utils/precheckrunner"
"github.com/jfrog/jfrog-cli-core/v2/artifactory/utils"
"github.com/jfrog/jfrog-cli-core/v2/utils/coreutils"
"github.com/jfrog/jfrog-cli-core/v2/utils/progressbar"
"github.com/jfrog/jfrog-client-go/artifactory"
Expand All @@ -30,6 +31,7 @@ const (
threadCount = 10
maxAllowedValLength = 2400
longPropertyCheckName = "Properties with value longer than 2.4K characters"
propertiesRequestTimeout = time.Minute * 30
)

// Property - Represents a property of an item
Expand All @@ -54,14 +56,15 @@ type FileWithLongProperty struct {
}

type LongPropertyCheck struct {
producerConsumer parallel.Runner
filesChan chan FileWithLongProperty
errorsQueue *clientutils.ErrorsQueue
repos []string
producerConsumer parallel.Runner
filesChan chan FileWithLongProperty
errorsQueue *clientutils.ErrorsQueue
repos []string
disabledDistinctiveAql bool
}

func NewLongPropertyCheck(repos []string) *LongPropertyCheck {
return &LongPropertyCheck{repos: repos}
func NewLongPropertyCheck(repos []string, disabledDistinctiveAql bool) *LongPropertyCheck {
return &LongPropertyCheck{repos: repos, disabledDistinctiveAql: disabledDistinctiveAql}
}

func (lpc *LongPropertyCheck) Name() string {
Expand Down Expand Up @@ -114,7 +117,7 @@ func (lpc *LongPropertyCheck) ExecuteCheck(args precheckrunner.RunArguments) (pa
// Returns the number of long properties found
func (lpc *LongPropertyCheck) longPropertiesTaskProducer(progress *progressbar.TasksProgressBar, args precheckrunner.RunArguments) int {
// Init
serviceManager, err := createTransferServiceManager(args.Context, args.ServerDetails)
serviceManager, err := utils.CreateServiceManagerWithContext(args.Context, args.ServerDetails, false, 0, retries, retriesWaitMilliSecs, propertiesRequestTimeout)
if err != nil {
return 0
}
Expand All @@ -131,7 +134,7 @@ func (lpc *LongPropertyCheck) longPropertiesTaskProducer(progress *progressbar.T
if long := isLongProperty(property); long {
log.Debug(fmt.Sprintf(`Found long property ('@%s':'%s')`, property.Key, property.Value))
if lpc.producerConsumer != nil {
_, _ = lpc.producerConsumer.AddTaskWithError(createSearchPropertyTask(property, lpc.repos, args, lpc.filesChan, progress), lpc.errorsQueue.AddError)
_, _ = lpc.producerConsumer.AddTaskWithError(lpc.createSearchPropertyTask(property, args, progress), lpc.errorsQueue.AddError)
}
if progress != nil {
progress.IncGeneralProgressTotalBy(1)
Expand Down Expand Up @@ -174,25 +177,25 @@ func getSearchAllPropertiesQuery(pageNumber int) string {

// Create a task that fetch from the server the files with the given property.
// We keep only the files that are at the requested repos and pass them at the files channel
func createSearchPropertyTask(property Property, repos []string, args precheckrunner.RunArguments, filesChan chan FileWithLongProperty, progress *progressbar.TasksProgressBar) parallel.TaskFunc {
func (lpc *LongPropertyCheck) createSearchPropertyTask(property Property, args precheckrunner.RunArguments, progress *progressbar.TasksProgressBar) parallel.TaskFunc {
return func(threadId int) (err error) {
serviceManager, err := createTransferServiceManager(args.Context, args.ServerDetails)
serviceManager, err := utils.CreateServiceManagerWithContext(args.Context, args.ServerDetails, false, 0, retries, retriesWaitMilliSecs, propertiesRequestTimeout)
if err != nil {
return
}
// Search
var query *servicesUtils.AqlSearchResult
if query, err = runSearchPropertyInFilesAql(serviceManager, property); err != nil {
if query, err = lpc.runSearchPropertyInFilesAql(serviceManager, property); err != nil {
return
}
log.Debug(fmt.Sprintf("[Thread=%d] Got %d files from the query", threadId, len(query.Results)))
for _, item := range query.Results {
file := api.FileRepresentation{Repo: item.Repo, Path: item.Path, Name: item.Name}
// Keep only if in the requested repos
if slices.Contains(repos, file.Repo) {
if slices.Contains(lpc.repos, file.Repo) {
fileWithLongProperty := FileWithLongProperty{file, property.valueLength(), property}
log.Debug(fmt.Sprintf("[Thread=%d] Found File{Repo=%s, Path=%s, Name=%s} with matching entry of long property.", threadId, file.Repo, file.Path, file.Name))
filesChan <- fileWithLongProperty
lpc.filesChan <- fileWithLongProperty
}
}
// Notify end of search for the current property
Expand All @@ -204,15 +207,17 @@ func createSearchPropertyTask(property Property, repos []string, args precheckru
}

// Get all the files that contains the given property using AQL
func runSearchPropertyInFilesAql(serviceManager artifactory.ArtifactoryServicesManager, property Property) (result *servicesUtils.AqlSearchResult, err error) {
func (lpc *LongPropertyCheck) runSearchPropertyInFilesAql(serviceManager artifactory.ArtifactoryServicesManager, property Property) (result *servicesUtils.AqlSearchResult, err error) {
result = &servicesUtils.AqlSearchResult{}
err = runAqlService(serviceManager, getSearchPropertyInFilesQuery(property), result)
err = runAqlService(serviceManager, lpc.getSearchPropertyInFilesQuery(property), result)
return
}

// Get the query that search files with specific property
func getSearchPropertyInFilesQuery(property Property) string {
return fmt.Sprintf(`items.find({"type": {"$eq":"any"},"@%s":"%s"}).include("repo","path","name")`, property.Key, property.Value)
func (lpc *LongPropertyCheck) getSearchPropertyInFilesQuery(property Property) string {
query := fmt.Sprintf(`items.find({"type": {"$eq":"any"},"@%s":"%s"}).include("repo","path","name")`, property.Key, property.Value)
query += appendDistinctIfNeeded(lpc.disabledDistinctiveAql)
return query
}

// Run AQL service that return a result in the given format structure 'v'
Expand Down
Loading

0 comments on commit d4e93a4

Please sign in to comment.