Skip to content

Commit 315c071

Browse files
cableheadcablehead
committed
wip
1 parent 6d447de commit 315c071

File tree

4 files changed

+249
-80
lines changed

4 files changed

+249
-80
lines changed

‎gpt/ctx.nu‎

Lines changed: 10 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -27,26 +27,15 @@ def frame-to-turn [frame: record] {
2727
let content_raw = if ($frame | get hash? | is-not-empty) { .cas $frame.hash }
2828
if ($content_raw | is-empty) { return }
2929

30-
let content = (
31-
if ($meta.type? == "document" and $meta.content_type? != null) {
32-
[
33-
{
34-
type: "document"
35-
source: {
36-
type: "base64"
37-
media_type: $meta.content_type
38-
data: ($content_raw | encode base64)
39-
}
40-
}
41-
]
42-
} else if (($meta | get content_type?) == "application/json") {
43-
$content_raw | from json
44-
} else {
45-
[
46-
{type: "text" text: $content_raw}
47-
]
48-
}
49-
)
30+
# Content is now stored as clean JSON (normalized format)
31+
let content = if (($meta | get content_type?) == "application/json") {
32+
$content_raw | from json
33+
} else {
34+
# Legacy fallback for old text-based storage
35+
[
36+
{type: "text" text: $content_raw}
37+
]
38+
}
5039

5140
{
5241
id: $frame.id
@@ -92,11 +81,10 @@ export def list [headish?] {
9281
export def resolve [headish?] {
9382
let turns = list $headish
9483
let options = $turns | get options? | compact --empty | if ($in | is-not-empty) {
95-
reduce {|it, acc| $acc | merge deep $it }
84+
reduce {|it acc| $acc | merge deep $it }
9685
} else { null }
9786
{
9887
messages: $turns
9988
options: $options
10089
}
10190
}
102-

‎gpt/mod.nu‎

Lines changed: 19 additions & 58 deletions
Original file line numberDiff line numberDiff line change
@@ -19,70 +19,28 @@ export use ./mcp.nu
1919
export use ./providers
2020
export use ./provider.nu
2121
export use ./prep.nu
22+
export use ./schema.nu
2223

2324
export def document [
2425
path: string # Path to the document file
2526
--name (-n): string # Optional name for the document (defaults to filename)
2627
--cache # Enable caching for this document
2728
--bookmark (-b): string # Bookmark this document registration
2829
] {
29-
# Validate file exists
30-
if not ($path | path exists) {
31-
error make {
32-
msg: $"File does not exist: ($path)"
33-
label: {
34-
text: "this path"
35-
span: (metadata $path).span
36-
}
37-
}
38-
}
30+
# Generate normalized document turn using schema layer
31+
let normalized_turn = schema document-turn $path {name: $name cache: $cache}
3932

40-
# Detect content type from file extension
41-
let content_type = match ($path | path parse | get extension | str downcase) {
42-
"pdf" => "application/pdf"
43-
"txt" => "text/plain"
44-
"md" => "text/markdown"
45-
"json" => "application/json"
46-
"csv" => "text/csv"
47-
"jpg"|"jpeg" => "image/jpeg"
48-
"png" => "image/png"
49-
"webp" => "image/webp"
50-
"gif" => "image/gif"
51-
"docx" => "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
52-
"xlsx" => "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
53-
_ => {
54-
let detected = (file --mime-type $path | split row ":" | get 1 | str trim)
55-
print $"Warning: Unknown extension, detected MIME type: ($detected)"
56-
$detected
57-
}
58-
}
33+
# Extract metadata for storage
34+
let metadata = $normalized_turn._metadata
35+
let clean_turn = $normalized_turn | reject _metadata
5936

60-
# Check file size (Anthropic has limits)
61-
let file_size = ($path | path expand | ls $in | get size | first)
62-
if $file_size > 100MB {
63-
# rough limit
64-
error make {
65-
msg: $"File too large: ($file_size). Consider splitting or compressing."
66-
}
67-
}
68-
69-
let document_name = $name | default ($path | path basename)
70-
let content = open $path --raw
71-
72-
let meta = {
73-
type: "document"
74-
content_type: $content_type
75-
role: "user"
76-
document_name: $document_name
77-
original_path: ($path | path expand)
78-
file_size: $file_size
79-
} | conditional-pipe $cache {
80-
insert cache true
81-
} | conditional-pipe ($bookmark | is-not-empty) {
37+
# Build storage metadata
38+
let meta = $metadata | insert role "user" | conditional-pipe ($bookmark | is-not-empty) {
8239
insert head $bookmark
8340
}
8441

85-
let turn = $content | .append gpt.turn --meta $meta
42+
# Store the clean normalized content
43+
let turn = $clean_turn.content | to json | .append gpt.turn --meta $meta
8644

8745
$turn
8846
}
@@ -100,11 +58,13 @@ export def main [
10058
] {
10159
let content = if $in == null {
10260
input "Enter prompt: "
103-
} else if ($in | describe) == "list<string>" {
104-
$in | str join $separator
10561
} else {
10662
$in
10763
}
64+
65+
# Generate normalized user turn using schema layer
66+
let normalized_turn = schema user-turn $content {json: $json cache: $cache separator: $separator}
67+
10868
let continues = $continues | append [] | each { ctx headish-to-id $in }
10969
let continues = $continues | conditional-pipe $respond { append (.head gpt.turn).id }
11070

@@ -114,7 +74,7 @@ export def main [
11474

11575
let meta = (
11676
{
117-
role: user
77+
role: $normalized_turn.role
11878
# options should be renamed to "inherited"
11979
options : (
12080
{}
@@ -124,12 +84,13 @@ export def main [
12484
)
12585
}
12686
| conditional-pipe ($head | is-not-empty) { insert head $head }
127-
| conditional-pipe $cache { insert cache true }
87+
| conditional-pipe ($normalized_turn.cache? == true) { insert cache true }
12888
| if $continues != null { insert continues $continues } else { }
12989
| if $json { insert content_type "application/json" } else { }
13090
)
13191

132-
let turn = $content | .append gpt.turn --meta $meta
92+
# Store the clean normalized content
93+
let turn = $normalized_turn.content | to json | .append gpt.turn --meta $meta
13394

13495
process-turn $turn
13596
}
@@ -239,7 +200,7 @@ export def process-turn [turn: record] {
239200

240201
# save the assistance response
241202
let turn = $content | to json | .append gpt.turn --meta $meta
242-
print $"THIS SHOULND'T BE POSSIBLE: ($turn)"
203+
print $"THIS SHOULND'T BE POSSIBLE: ($turn)"
243204
$content | process-turn-response $turn
244205
}
245206

‎gpt/schema.nu‎

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
# Schema generation layer - creates clean normalized turns and context windows
2+
# This ensures consistent format between CLI commands and test fixtures
3+
4+
# Generate a normalized user turn from content and options
5+
export def user-turn [
6+
content: any
7+
options?: record = {} # {json?: bool, cache?: bool, separator?: string}
8+
] {
9+
let json = $options.json? | default false
10+
let cache = $options.cache? | default false
11+
let separator = $options.separator? | default "\n\n---\n\n"
12+
13+
let processed_content = if ($content | describe) == "list<string>" {
14+
$content | str join $separator
15+
} else {
16+
$content
17+
}
18+
19+
let content_blocks = if $json {
20+
$processed_content | from json
21+
} else {
22+
[
23+
{type: "text" text: $processed_content}
24+
]
25+
}
26+
27+
{
28+
role: "user"
29+
content: $content_blocks
30+
} | if $cache {
31+
insert cache $cache
32+
} else { $in }
33+
}
34+
35+
# Generate a normalized document turn from file path and options
36+
export def document-turn [
37+
path: string
38+
options?: record = {} # {name?: string, cache?: bool}
39+
] {
40+
let name = $options.name?
41+
let cache = $options.cache? | default false
42+
43+
# Validate file exists
44+
if not ($path | path exists) {
45+
error make {
46+
msg: $"File does not exist: ($path)"
47+
label: {
48+
text: "this path"
49+
span: (metadata $path).span
50+
}
51+
}
52+
}
53+
54+
# Detect content type from file extension
55+
let content_type = match ($path | path parse | get extension | str downcase) {
56+
"pdf" => "application/pdf"
57+
"txt" => "text/plain"
58+
"md" => "text/markdown"
59+
"json" => "application/json"
60+
"csv" => "text/csv"
61+
"jpg"|"jpeg" => "image/jpeg"
62+
"png" => "image/png"
63+
"webp" => "image/webp"
64+
"gif" => "image/gif"
65+
"docx" => "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
66+
"xlsx" => "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
67+
_ => {
68+
let detected = (file --mime-type $path | split row ":" | get 1 | str trim)
69+
print $"Warning: Unknown extension, detected MIME type: ($detected)"
70+
$detected
71+
}
72+
}
73+
74+
# Check file size (rough limit)
75+
let file_size = ($path | path expand | ls $in | get size | first)
76+
if $file_size > 100MB {
77+
error make {
78+
msg: $"File too large: ($file_size). Consider splitting or compressing."
79+
}
80+
}
81+
82+
let document_name = $name | default ($path | path basename)
83+
let content = open $path --raw | encode base64
84+
85+
let content_blocks = [
86+
{
87+
type: "document"
88+
source: {
89+
type: "base64"
90+
media_type: $content_type
91+
data: $content
92+
}
93+
}
94+
]
95+
96+
{
97+
role: "user"
98+
content: $content_blocks
99+
# Store metadata for internal use (not part of normalized schema)
100+
_metadata: {
101+
document_name: $document_name
102+
original_path: ($path | path expand)
103+
file_size: $file_size
104+
type: "document"
105+
content_type: $content_type
106+
}
107+
} | if $cache {
108+
insert cache $cache
109+
} else { $in }
110+
}
111+
112+
# Helper to conditionally apply transformations
113+
def conditional-pipe [
114+
condition: bool
115+
action: closure
116+
] {
117+
if $condition { do $action } else { $in }
118+
}

0 commit comments

Comments
 (0)