Allow skipping JSON check and add JSONLines

Author: svilenmarkov

docs/configuration.md

@@ -1297,6 +1297,7 @@ Examples:
 | body | any | no | |
 | frameless | boolean | no | false |
 | allow-insecure | boolean | no | false |
+| skip-json-validation | boolean | no | false |
 | template | string | yes | |
 | parameters | key (string) & value (string|array) | no | |
 | subrequests | map of requests | no | |
@@ -1344,6 +1345,9 @@ When set to `true`, removes the border and padding around the widget.
 ##### `allow-insecure`
 Whether to ignore invalid/self-signed certificates.
 
+##### `skip-json-validation`
+When set to `true`, skips the JSON validation step. This is useful when the API returns JSON Lines/newline-delimited JSON, which is a format that consists of several JSON objects separated by newlines.
+
 ##### `template`
 The template that will be used to display the data. It relies on Go's `html/template` package so it's recommended to go through [its documentation](https://pkg.go.dev/text/template) to understand how to do basic things such as conditionals, loops, etc. In addition, it also uses [tidwall's gjson](https://github.com/tidwall/gjson) package to parse the JSON data so it's worth going through its documentation if you want to use more advanced JSON selectors. You can view additional examples with explanations and function definitions [here](custom-api.md).
 

docs/custom-api.md

@@ -309,6 +309,55 @@ You can also access the response headers:
 <div>{{ .Response.Header.Get "Content-Type" }}</div>
 ```
 
+<hr>
+
+JSON response:
+
+```json
+{"name": "Steve", "age": 30}
+{"name": "Alex", "age": 25}
+{"name": "John", "age": 35}
+```
+
+The above format is "[ndjson](https://docs.mulesoft.com/dataweave/latest/dataweave-formats-ndjson)" or "[JSON Lines](https://jsonlines.org/)", where each line is a separate JSON object. To parse this format, you must first disable the JSON validation check in your config, since by default the response is expected to be a single valid JSON object:
+
+```yaml
+- type: custom-api
+  skip-json-validation: true
+```
+
+Then, to iterate over each object you can use `.JSONLines`:
+
+```html
+{{ range .JSONLines }}
+  <p>{{ .String "name" }} is {{ .Int "age" }} years old</p>
+{{ end }}
+```
+
+Output:
+
+```html
+<p>Steve is 30 years old</p>
+<p>Alex is 25 years old</p>
+<p>John is 35 years old</p>
+```
+
+For other ways of selecting data from a JSON Lines response, have a look at the docs for [tidwall/gjson](https://github.com/tidwall/gjson/tree/master?tab=readme-ov-file#json-lines). For example, to  get an array of all names, you can use the following:
+
+```html
+{{ range .JSON.Array "..#.name" }}
+  <p>{{ .String "" }}</p>
+{{ end }}
+```
+
+Output:
+
+```html
+<p>Steve</p>
+<p>Alex</p>
+<p>John</p>
+```
+
 ## Functions
 
 The following functions are available on the `JSON` object:

internal/glance/widget-custom-api.go

@@ -25,15 +25,16 @@ var customAPIWidgetTemplate = mustParseTemplate("custom-api.html", "widget-base.
 
 // Needs to be exported for the YAML unmarshaler to work
 type CustomAPIRequest struct {
-	URL           string               `yaml:"url"`
-	AllowInsecure bool                 `yaml:"allow-insecure"`
-	Headers       map[string]string    `yaml:"headers"`
-	Parameters    queryParametersField `yaml:"parameters"`
-	Method        string               `yaml:"method"`
-	BodyType      string               `yaml:"body-type"`
-	Body          any                  `yaml:"body"`
-	bodyReader    io.ReadSeeker        `yaml:"-"`
-	httpRequest   *http.Request        `yaml:"-"`
+	URL                string               `yaml:"url"`
+	AllowInsecure      bool                 `yaml:"allow-insecure"`
+	Headers            map[string]string    `yaml:"headers"`
+	Parameters         queryParametersField `yaml:"parameters"`
+	Method             string               `yaml:"method"`
+	BodyType           string               `yaml:"body-type"`
+	Body               any                  `yaml:"body"`
+	SkipJSONValidation bool                 `yaml:"skip-json-validation"`
+	bodyReader         io.ReadSeeker        `yaml:"-"`
+	httpRequest        *http.Request        `yaml:"-"`
 }
 
 type customAPIWidget struct {
@@ -157,6 +158,17 @@ type customAPITemplateData struct {
 	subrequests map[string]*customAPIResponseData
 }
 
+func (data *customAPITemplateData) JSONLines() []decoratedGJSONResult {
+	result := make([]decoratedGJSONResult, 0, 5)
+
+	gjson.ForEachLine(data.JSON.Raw, func(line gjson.Result) bool {
+		result = append(result, decoratedGJSONResult{line})
+		return true
+	})
+
+	return result
+}
+
 func (data *customAPITemplateData) Subrequest(key string) *customAPIResponseData {
 	req, exists := data.subrequests[key]
 	if !exists {
@@ -190,7 +202,7 @@ func fetchCustomAPIRequest(ctx context.Context, req *CustomAPIRequest) (*customA
 
 	body := strings.TrimSpace(string(bodyBytes))
 
-	if body != "" && !gjson.Valid(body) {
+	if !req.SkipJSONValidation && body != "" && !gjson.Valid(body) {
 		truncatedBody, isTruncated := limitStringLength(body, 100)
 		if isTruncated {
 			truncatedBody += "... <truncated>"