feat: detected field values (#14350)

cmd/logcli/main.go

@@ -692,7 +692,7 @@ func newVolumeQuery(rangeQuery bool, cmd *kingpin.CmdClause) *volume.Query {
 
 func newDetectedFieldsQuery(cmd *kingpin.CmdClause) *detected.FieldsQuery {
 	// calculate query range from cli params
-	var from, to string
+	var fieldName, from, to string
 	var since time.Duration
 
 	q := &detected.FieldsQuery{}
@@ -705,24 +705,28 @@ func newDetectedFieldsQuery(cmd *kingpin.CmdClause) *detected.FieldsQuery {
 		q.Start = mustParse(from, defaultStart)
 		q.End = mustParse(to, defaultEnd)
 
+		q.FieldName = fieldName
+
 		q.Quiet = *quiet
 
 		return nil
 	})
 
-	cmd.Flag("field-limit", "Limit on number of fields to return.").
+	cmd.Flag("limit", "Limit on number of fields or values to return.").
 		Default("100").
-		IntVar(&q.FieldLimit)
+		IntVar(&q.Limit)
 	cmd.Flag("line-limit", "Limit the number of lines each subquery is allowed to process.").
 		Default("1000").
 		IntVar(&q.LineLimit)
 	cmd.Arg("query", "eg '{foo=\"bar\",baz=~\".*blip\"} |~ \".*error.*\"'").
 		Required().
 		StringVar(&q.QueryString)
+	cmd.Arg("field", "The name of the field.").Default("").StringVar(&fieldName)
 	cmd.Flag("since", "Lookback window.").Default("1h").DurationVar(&since)
 	cmd.Flag("from", "Start looking for logs at this absolute time (inclusive)").StringVar(&from)
 	cmd.Flag("to", "Stop looking for logs at this absolute time (exclusive)").StringVar(&to)
 	cmd.Flag("step", "Query resolution step width, for metric queries. Evaluate the query at the specified step over the time range.").
+		Default("10s").
 		DurationVar(&q.Step)
 
 	return q

docs/sources/setup/install/helm/reference.md

@@ -5640,6 +5640,7 @@ null
       "/loki/api/v1/index/volume",
       "/loki/api/v1/index/volume_range",
       "/loki/api/v1/format_query",
+      "/loki/api/v1/detected_field",
       "/loki/api/v1/detected_fields",
       "/loki/api/v1/detected_labels",
       "/loki/api/v1/patterns"
@@ -5702,6 +5703,7 @@ null
   "/loki/api/v1/index/volume",
   "/loki/api/v1/index/volume_range",
   "/loki/api/v1/format_query",
+  "/loki/api/v1/detected_field",
   "/loki/api/v1/detected_fields",
   "/loki/api/v1/detected_labels",
   "/loki/api/v1/patterns"

pkg/ingester-rf1/ingester.go

@@ -881,6 +881,6 @@ func (i *Ingester) GetDetectedFields(_ context.Context, r *logproto.DetectedFiel
 				Cardinality: 1,
 			},
 		},
-		FieldLimit: r.GetFieldLimit(),
+		Limit: r.GetLimit(),
 	}, nil
 }

pkg/ingester/ingester.go

@@ -1595,7 +1595,7 @@ func (i *Ingester) GetDetectedFields(_ context.Context, r *logproto.DetectedFiel
 				Cardinality: 1,
 			},
 		},
-		FieldLimit: r.GetFieldLimit(),
+		Limit: r.GetLimit(),
 	}, nil
 }
 

pkg/logcli/client/client.go

@@ -28,17 +28,18 @@ import (
 )
 
 const (
-	queryPath          = "/loki/api/v1/query"
-	queryRangePath     = "/loki/api/v1/query_range"
-	labelsPath         = "/loki/api/v1/labels"
-	labelValuesPath    = "/loki/api/v1/label/%s/values"
-	seriesPath         = "/loki/api/v1/series"
-	tailPath           = "/loki/api/v1/tail"
-	statsPath          = "/loki/api/v1/index/stats"
-	volumePath         = "/loki/api/v1/index/volume"
-	volumeRangePath    = "/loki/api/v1/index/volume_range"
-	detectedFieldsPath = "/loki/api/v1/detected_fields"
-	defaultAuthHeader  = "Authorization"
+	queryPath               = "/loki/api/v1/query"
+	queryRangePath          = "/loki/api/v1/query_range"
+	labelsPath              = "/loki/api/v1/labels"
+	labelValuesPath         = "/loki/api/v1/label/%s/values"
+	seriesPath              = "/loki/api/v1/series"
+	tailPath                = "/loki/api/v1/tail"
+	statsPath               = "/loki/api/v1/index/stats"
+	volumePath              = "/loki/api/v1/index/volume"
+	volumeRangePath         = "/loki/api/v1/index/volume_range"
+	detectedFieldsPath      = "/loki/api/v1/detected_fields"
+	detectedFieldValuesPath = "/loki/api/v1/detected_field/%s/values"
+	defaultAuthHeader       = "Authorization"
 
 	// HTTP header keys
 	HTTPScopeOrgID          = "X-Scope-OrgID"
@@ -61,7 +62,7 @@ type Client interface {
 	GetStats(queryStr string, start, end time.Time, quiet bool) (*logproto.IndexStatsResponse, error)
 	GetVolume(query *volume.Query) (*loghttp.QueryResponse, error)
 	GetVolumeRange(query *volume.Query) (*loghttp.QueryResponse, error)
-	GetDetectedFields(queryStr string, fieldLimit, lineLimit int, start, end time.Time, step time.Duration, quiet bool) (*loghttp.DetectedFieldsResponse, error)
+	GetDetectedFields(queryStr, fieldName string, fieldLimit, lineLimit int, start, end time.Time, step time.Duration, quiet bool) (*loghttp.DetectedFieldsResponse, error)
 }
 
 // Tripperware can wrap a roundtripper.
@@ -234,15 +235,16 @@ func (c *DefaultClient) getVolume(path string, query *volume.Query) (*loghttp.Qu
 }
 
 func (c *DefaultClient) GetDetectedFields(
-	queryStr string,
-	fieldLimit, lineLimit int,
+	queryStr, fieldName string,
+	limit, lineLimit int,
 	start, end time.Time,
 	step time.Duration,
 	quiet bool,
 ) (*loghttp.DetectedFieldsResponse, error) {
+
 	qsb := util.NewQueryStringBuilder()
 	qsb.SetString("query", queryStr)
-	qsb.SetInt("field_limit", int64(fieldLimit))
+	qsb.SetInt("limit", int64(limit))
 	qsb.SetInt("line_limit", int64(lineLimit))
 	qsb.SetInt("start", start.UnixNano())
 	qsb.SetInt("end", end.UnixNano())
@@ -251,7 +253,12 @@ func (c *DefaultClient) GetDetectedFields(
 	var err error
 	var r loghttp.DetectedFieldsResponse
 
-	if err = c.doRequest(detectedFieldsPath, qsb.Encode(), quiet, &r); err != nil {
+	path := detectedFieldsPath
+	if fieldName != "" {
+		path = fmt.Sprintf(detectedFieldValuesPath, url.PathEscape(fieldName))
+	}
+
+	if err = c.doRequest(path, qsb.Encode(), quiet, &r); err != nil {
 		return nil, err
 	}
 

pkg/logcli/client/file.go

@@ -207,7 +207,7 @@ func (f *FileClient) GetVolumeRange(_ *volume.Query) (*loghttp.QueryResponse, er
 }
 
 func (f *FileClient) GetDetectedFields(
-	_ string,
+	_, _ string,
 	_, _ int,
 	_, _ time.Time,
 	_ time.Duration,

pkg/logcli/detected/fields.go

@@ -18,10 +18,11 @@ type FieldsQuery struct {
 	QueryString   string
 	Start         time.Time
 	End           time.Time
-	FieldLimit    int
+	Limit         int
 	LineLimit     int
 	Step          time.Duration
 	Quiet         bool
+	FieldName     string
 	ColoredOutput bool
 }
 
@@ -30,7 +31,16 @@ func (q *FieldsQuery) Do(c client.Client, outputMode string) {
 	var resp *loghttp.DetectedFieldsResponse
 	var err error
 
-	resp, err = c.GetDetectedFields(q.QueryString, q.FieldLimit, q.LineLimit, q.Start, q.End, q.Step, q.Quiet)
+	resp, err = c.GetDetectedFields(
+		q.QueryString,
+		q.FieldName,
+		q.Limit,
+		q.LineLimit,
+		q.Start,
+		q.End,
+		q.Step,
+		q.Quiet,
+	)
 	if err != nil {
 		log.Fatalf("Error doing request: %+v", err)
 	}
@@ -43,12 +53,17 @@ func (q *FieldsQuery) Do(c client.Client, outputMode string) {
 		}
 		fmt.Println(string(out))
 	default:
-		output := make([]string, len(resp.Fields))
-		for i, field := range resp.Fields {
-			bold := color.New(color.Bold)
-			output[i] = fmt.Sprintf("label: %s\t\t", bold.Sprintf("%s", field.Label)) +
-				fmt.Sprintf("type: %s\t\t", bold.Sprintf("%s", field.Type)) +
-				fmt.Sprintf("cardinality: %s", bold.Sprintf("%d", field.Cardinality))
+		var output []string
+		if len(resp.Fields) > 0 {
+			output = make([]string, len(resp.Fields))
+			for i, field := range resp.Fields {
+				bold := color.New(color.Bold)
+				output[i] = fmt.Sprintf("label: %s\t\t", bold.Sprintf("%s", field.Label)) +
+					fmt.Sprintf("type: %s\t\t", bold.Sprintf("%s", field.Type)) +
+					fmt.Sprintf("cardinality: %s", bold.Sprintf("%d", field.Cardinality))
+			}
+		} else if len(resp.Values) > 0 {
+			output = resp.Values
 		}
 
 		slices.Sort(output)

pkg/logcli/query/query_test.go

@@ -486,7 +486,7 @@ func (t *testQueryClient) GetVolumeRange(_ *volume.Query) (*loghttp.QueryRespons
 }
 
 func (t *testQueryClient) GetDetectedFields(
-	_ string,
+	_, _ string,
 	_, _ int,
 	_, _ time.Time,
 	_ time.Duration,

pkg/loghttp/detected.go

@@ -5,6 +5,7 @@ import "github.com/grafana/loki/v3/pkg/logproto"
 // LabelResponse represents the http json response to a label query
 type DetectedFieldsResponse struct {
 	Fields []DetectedField `json:"fields,omitempty"`
+	Values []string        `json:"values,omitempty"`
 }
 
 type DetectedField struct {

pkg/loghttp/params.go

@@ -19,7 +19,7 @@ import (
 
 const (
 	defaultQueryLimit = 100
-	defaultFieldLimit = 1000
+	defaultLimit      = 1000
 	defaultSince      = 1 * time.Hour
 	defaultDirection  = logproto.BACKWARD
 )
@@ -46,11 +46,18 @@ func lineLimit(r *http.Request) (uint32, error) {
 	return uint32(l), nil
 }
 
-func fieldLimit(r *http.Request) (uint32, error) {
-	l, err := parseInt(r.Form.Get("field_limit"), defaultFieldLimit)
+func detectedFieldsLimit(r *http.Request) (uint32, error) {
+	limit := r.Form.Get("limit")
+	if limit == "" {
+		// for backwards compatability
+		limit = r.Form.Get("field_limit")
+	}
+
+	l, err := parseInt(limit, defaultLimit)
 	if err != nil {
 		return 0, err
 	}
+
 	if l <= 0 {
 		return 0, errors.New("limit must be a positive value")
 	}

pkg/loghttp/query.go

@@ -9,6 +9,7 @@ import (
 	"unsafe"
 
 	"github.com/c2h5oh/datasize"
+	"github.com/gorilla/mux"
 	"github.com/grafana/jsonparser"
 	json "github.com/json-iterator/go"
 	"github.com/prometheus/common/model"
@@ -650,6 +651,7 @@ func ParseDetectedFieldsQuery(r *http.Request) (*logproto.DetectedFieldsRequest,
 	result := &logproto.DetectedFieldsRequest{}
 
 	result.Query = query(r)
+	result.Values, result.Name = values(r)
 	result.Start, result.End, err = bounds(r)
 	if err != nil {
 		return nil, err
@@ -664,7 +666,7 @@ func ParseDetectedFieldsQuery(r *http.Request) (*logproto.DetectedFieldsRequest,
 		return nil, err
 	}
 
-	result.FieldLimit, err = fieldLimit(r)
+	result.Limit, err = detectedFieldsLimit(r)
 	if err != nil {
 		return nil, err
 	}
@@ -684,9 +686,15 @@ func ParseDetectedFieldsQuery(r *http.Request) (*logproto.DetectedFieldsRequest,
 	if (result.End.Sub(result.Start) / step) > 11000 {
 		return nil, errStepTooSmall
 	}
+
 	return result, nil
 }
 
+func values(r *http.Request) (bool, string) {
+	name, ok := mux.Vars(r)["name"]
+	return ok, name
+}
+
 func targetLabels(r *http.Request) []string {
 	lbls := strings.Split(r.Form.Get("targetLabels"), ",")
 	if (len(lbls) == 1 && lbls[0] == "") || len(lbls) == 0 {

pkg/logproto/compat.go

@@ -524,7 +524,7 @@ func (m *DetectedFieldsRequest) LogToSpan(sp opentracing.Span) {
 		otlog.String("start", m.Start.String()),
 		otlog.String("end", m.End.String()),
 		otlog.String("step", time.Duration(m.Step).String()),
-		otlog.String("field_limit", fmt.Sprintf("%d", m.FieldLimit)),
+		otlog.String("field_limit", fmt.Sprintf("%d", m.Limit)),
 		otlog.String("line_limit", fmt.Sprintf("%d", m.LineLimit)),
 	}
 	sp.LogFields(fields...)