Skip to content

Commit c885d26

Browse files
nnaydenownnaydenow
authored
chore: generate llms.txt (#13207)
1 parent cd993af commit c885d26

File tree

3 files changed

+373
-1
lines changed

3 files changed

+373
-1
lines changed

‎packages/website/.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ src/components/Editor/ui5-autocomplete.json
2020
/illustrations-tnt
2121
static/packages
2222
static/assets
23+
static/llms.txt
2324

2425
# Generated files
2526
.docusaurus

‎packages/website/build-scripts/llms-txt-generation/index.mjs‎

Lines changed: 370 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,370 @@
1+
import fs from "fs";
2+
import path from "path";
3+
import { fileURLToPath } from 'url';
4+
5+
const __filename = fileURLToPath(import.meta.url);
6+
const __dirname = path.dirname(__filename);
7+
8+
const PACKAGES = ["main", "fiori", "compat", "ai"];
9+
10+
// Clean component description
11+
function cleanDescription(description) {
12+
if (!description) return '';
13+
14+
// Split into lines
15+
const lines = description.split('\n').map(l => l.trim()).filter(l => l.length > 0);
16+
17+
// Find first meaningful line (skip headings like "### Overview")
18+
let firstMeaningfulLine = '';
19+
for (const line of lines) {
20+
// Skip markdown headings
21+
if (line.match(/^#+\s/)) {
22+
continue;
23+
}
24+
// Skip lines that are just "Overview" or similar
25+
if (line.toLowerCase() === 'overview' || line.toLowerCase() === 'usage') {
26+
continue;
27+
}
28+
firstMeaningfulLine = line;
29+
break;
30+
}
31+
32+
if (!firstMeaningfulLine) return '';
33+
34+
// Remove HTML tags and backticks
35+
let cleaned = firstMeaningfulLine.replace(/<[^>]*>/g, '').replace(/`/g, '');
36+
37+
// Limit length
38+
if (cleaned.length > 150) {
39+
return cleaned.substring(0, 150).trim() + '...';
40+
}
41+
42+
return cleaned;
43+
}
44+
45+
// Get components from custom-elements manifest
46+
function getComponentsFromManifests() {
47+
const componentsByPackage = {};
48+
let foundAnyManifest = false;
49+
50+
PACKAGES.forEach(packageName => {
51+
const manifestPath = path.resolve(__dirname, `../../../${packageName}/dist/custom-elements-internal.json`);
52+
53+
if (!fs.existsSync(manifestPath)) {
54+
console.warn(`⚠ Manifest not found for ${packageName} package: ${manifestPath}`);
55+
return;
56+
}
57+
58+
foundAnyManifest = true;
59+
const manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf-8'));
60+
const components = [];
61+
62+
manifest.modules?.forEach(module => {
63+
module.declarations?.forEach(declaration => {
64+
if (declaration.kind === "class" &&
65+
declaration.customElement &&
66+
declaration.tagName &&
67+
declaration._ui5privacy === "public") {
68+
components.push({
69+
name: declaration.name,
70+
tagName: declaration.tagName,
71+
description: cleanDescription(declaration.description)
72+
});
73+
}
74+
});
75+
});
76+
77+
if (components.length > 0) {
78+
componentsByPackage[packageName] = components.sort((a, b) => a.name.localeCompare(b.name));
79+
}
80+
});
81+
82+
if (!foundAnyManifest) {
83+
console.warn('⚠ No component manifests found. Run "yarn build" from the root directory first.');
84+
}
85+
86+
return componentsByPackage;
87+
}
88+
89+
// Get all markdown files recursively
90+
function getMarkdownFiles(dir, fileList = []) {
91+
const files = fs.readdirSync(dir);
92+
93+
files.forEach(file => {
94+
const filePath = path.join(dir, file);
95+
if (fs.statSync(filePath).isDirectory()) {
96+
getMarkdownFiles(filePath, fileList);
97+
} else if (file.endsWith('.md') && file !== 'README.md') {
98+
fileList.push(filePath);
99+
}
100+
});
101+
102+
return fileList;
103+
}
104+
105+
// Extract title from markdown file
106+
function extractTitle(filePath) {
107+
const content = fs.readFileSync(filePath, 'utf-8');
108+
109+
// Try to find title in frontmatter
110+
const frontmatterMatch = content.match(/^---\n[\s\S]*?title:\s*(.+?)\n[\s\S]*?---/);
111+
if (frontmatterMatch) {
112+
return frontmatterMatch[1].replace(/['"]/g, '');
113+
}
114+
115+
// Try to find first h1 heading
116+
const h1Match = content.match(/^#\s+(.+)$/m);
117+
if (h1Match) {
118+
return h1Match[1];
119+
}
120+
121+
// Fallback to filename
122+
return path.basename(filePath, '.md').replace(/[-_]/g, ' ');
123+
}
124+
125+
// Convert file path to URL path
126+
function toUrlPath(filePath, baseDir) {
127+
const relativePath = path.relative(baseDir, filePath)
128+
.replace(/\\/g, '/')
129+
.replace(/\.md$/, '')
130+
.replace(/\/README$/, '');
131+
132+
// Remove numeric prefixes (e.g., "01-first-steps" -> "first-steps")
133+
const cleanPath = relativePath.replace(/\/?\d+-/g, '/').replace(/^\//, '');
134+
135+
return cleanPath;
136+
}
137+
138+
// Extract category name from folder name (e.g., "1-getting-started" -> "Getting Started")
139+
function extractCategoryName(folderName) {
140+
// Remove number prefix and convert to title case
141+
const withoutNumber = folderName.replace(/^\d+-/, '');
142+
return withoutNumber
143+
.split('-')
144+
.map(word => word.charAt(0).toUpperCase() + word.slice(1))
145+
.join(' ');
146+
}
147+
148+
// Group files by category based on folder structure
149+
function categorizeFiles(files, baseDir) {
150+
const categories = new Map();
151+
152+
files.forEach(filePath => {
153+
const relativePath = path.relative(baseDir, filePath);
154+
const title = extractTitle(filePath);
155+
const urlPath = toUrlPath(filePath, baseDir);
156+
157+
// Get the first directory in the path
158+
const pathParts = relativePath.split(path.sep);
159+
const firstDir = pathParts[0];
160+
161+
// Skip framework integration files
162+
if (relativePath.includes('3-frameworks')) {
163+
return;
164+
}
165+
166+
// Skip internal docs
167+
if (relativePath.includes('internal')) {
168+
return;
169+
}
170+
171+
// Determine category
172+
let categoryName = 'General';
173+
174+
if (firstDir.match(/^\d+-/)) {
175+
// Folder or file with number prefix (e.g., "1-getting-started" or "08-Releases.md")
176+
if (firstDir.endsWith('.md')) {
177+
// Root level file with number prefix - put in General
178+
categoryName = 'General';
179+
} else {
180+
// Directory with number prefix
181+
categoryName = extractCategoryName(firstDir);
182+
}
183+
} else if (firstDir.includes('.md')) {
184+
// Root level file without number prefix
185+
categoryName = 'General';
186+
} else if (pathParts.length > 1 && pathParts[0] !== 'images') {
187+
// File in a subdirectory (non-numbered)
188+
categoryName = 'Other';
189+
}
190+
191+
if (!categories.has(categoryName)) {
192+
categories.set(categoryName, []);
193+
}
194+
195+
categories.get(categoryName).push({ title, path: urlPath });
196+
});
197+
198+
// Convert Map to object and sort categories
199+
// Keep numbered folders in order, put General last
200+
const sortedCategories = {};
201+
const categoryKeys = Array.from(categories.keys()).sort((a, b) => {
202+
// Extract number prefix if exists
203+
const getOrder = (name) => {
204+
if (name === 'General') return 999;
205+
if (name === 'Other') return 1000;
206+
// Try to match the original folder order (1-getting-started, 2-advanced, etc.)
207+
const orderMap = {
208+
'Getting Started': 1,
209+
'Advanced': 2,
210+
'Development': 4,
211+
'Contributing': 5,
212+
'Migration Guides': 6
213+
};
214+
return orderMap[name] || 500;
215+
};
216+
return getOrder(a) - getOrder(b);
217+
});
218+
219+
categoryKeys.forEach(key => {
220+
sortedCategories[key] = categories.get(key);
221+
});
222+
223+
return sortedCategories;
224+
}
225+
226+
// Generate llms.txt content
227+
function generateLlmsTxt() {
228+
const docsDir = path.join(__dirname, '../../docs/docs');
229+
const packageJson = JSON.parse(fs.readFileSync(path.join(__dirname, '../../package.json'), 'utf-8'));
230+
const componentsByPackage = getComponentsFromManifests();
231+
232+
if (!fs.existsSync(docsDir)) {
233+
console.error('Documentation directory not found. Please run "yarn generate-documentation" first.');
234+
process.exit(1);
235+
}
236+
237+
const files = getMarkdownFiles(docsDir);
238+
const categories = categorizeFiles(files, docsDir);
239+
240+
let content = `# UI5 Web Components
241+
242+
> Enterprise-grade, framework-agnostic web components library implementing SAP Fiori design
243+
244+
Version: ${packageJson.version}
245+
Website: https://ui5.github.io/webcomponents/
246+
GitHub: https://github.com/SAP/ui5-webcomponents
247+
248+
## Overview
249+
250+
UI5 Web Components is a comprehensive library of reusable UI elements designed for building modern web applications. Based on web standards, these components work with any framework (React, Angular, Vue) or vanilla JavaScript.
251+
252+
### Key Features
253+
254+
- **Framework-agnostic**: Works with React, Angular, Vue, or plain JavaScript
255+
- **Enterprise-ready**: Production-tested components following SAP Fiori design guidelines
256+
- **Accessible**: WCAG 2.1 compliant with full keyboard navigation and screen reader support
257+
- **Themable**: Multiple built-in themes with customization options
258+
- **TypeScript support**: Full type definitions included
259+
- **i18n**: Multi-language support with 40+ locales
260+
261+
### Quick Start
262+
263+
\`\`\`bash
264+
npm install @ui5/webcomponents
265+
\`\`\`
266+
267+
\`\`\`javascript
268+
import "@ui5/webcomponents/dist/Button.js";
269+
\`\`\`
270+
271+
\`\`\`html
272+
<ui5-button>Click Me</ui5-button>
273+
\`\`\`
274+
275+
## Components
276+
277+
`;
278+
279+
// Add component links by package
280+
Object.entries(componentsByPackage).forEach(([packageName, components]) => {
281+
const packageDisplayName = packageName === 'main' ? 'Main Components' :
282+
packageName === 'fiori' ? 'Fiori Components' :
283+
packageName === 'compat' ? 'Compat Components' :
284+
packageName === 'ai' ? 'AI Components' : packageName;
285+
286+
content += `### ${packageDisplayName}\n\n`;
287+
288+
components.forEach(component => {
289+
let url;
290+
if (packageName === 'main') {
291+
// Main package uses the simpler URL
292+
url = `https://ui5.github.io/webcomponents/components/${component.name}/`;
293+
} else {
294+
// Other packages use the nightly URL with package name
295+
url = `https://ui5.github.io/webcomponents/nightly/components/${packageName}/${component.name}/`;
296+
}
297+
298+
// Show description if it's meaningful (not empty)
299+
const description = component.description ? ` - ${component.description}` : '';
300+
content += `- [${component.name}](${url})${description}\n`;
301+
});
302+
content += '\n';
303+
});
304+
305+
content += `## Documentation
306+
307+
`;
308+
309+
// Add categorized documentation links
310+
Object.entries(categories).forEach(([category, links]) => {
311+
content += `### ${category}\n\n`;
312+
links.forEach(link => {
313+
content += `- [${link.title}](https://ui5.github.io/webcomponents/docs/${link.path}/)\n`;
314+
});
315+
content += '\n';
316+
});
317+
318+
// Add additional important links
319+
content += `## Additional Resources
320+
321+
- [Component Catalog](https://ui5.github.io/webcomponents/components/)
322+
- [Icon Explorer](https://ui5.github.io/webcomponents/icons/)
323+
- [Playground](https://ui5.github.io/webcomponents/play/)
324+
- [API Reference](https://ui5.github.io/webcomponents/components/main/)
325+
- [Changelog](https://github.com/SAP/ui5-webcomponents/releases)
326+
- [FAQ](https://ui5.github.io/webcomponents/docs/FAQ/)
327+
328+
## Common Patterns
329+
330+
### Importing Components
331+
332+
\`\`\`javascript
333+
// Main components
334+
import "@ui5/webcomponents/dist/Button.js";
335+
import "@ui5/webcomponents/dist/Input.js";
336+
337+
// Fiori components
338+
import "@ui5/webcomponents-fiori/dist/ShellBar.js";
339+
340+
// Icons (always use dist path)
341+
import "@ui5/webcomponents-icons/dist/add.js";
342+
\`\`\`
343+
344+
## Important Notes for AI Assistants
345+
346+
- Always import from \`/dist/\` path, not from package root
347+
- Component tags use lowercase with hyphens (e.g., \`ui5-button\`, not \`UI5Button\`)
348+
- Icons must be imported separately: \`import "@ui5/webcomponents-icons/dist/icon-name.js"\`
349+
- Use attribute selectors in CSS: \`[ui5-button]\`, not \`ui5-button\`
350+
- Events are native DOM events, use standard \`.addEventListener()\`
351+
- For TypeScript, import types from package root: \`import Button from "@ui5/webcomponents/dist/Button.js"\`
352+
353+
---
354+
355+
Generated: ${new Date().toISOString()}
356+
`;
357+
358+
return content;
359+
}
360+
361+
// Write llms.txt to static directory
362+
function writeLlmsTxt() {
363+
const content = generateLlmsTxt();
364+
const outputPath = path.join(__dirname, '../../static/llms.txt');
365+
366+
fs.writeFileSync(outputPath, content, 'utf-8');
367+
console.log('✓ Generated llms.txt');
368+
}
369+
370+
writeLlmsTxt();

‎packages/website/package.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,8 @@
88
"generate-documentation": "rimraf ./docs/docs && node ./build-scripts/documentation-generation/index.mjs",
99
"generate-icons": "rimraf ./icons && rimraf ./icons-tnt && rimraf ./icons-business-suite && node ./build-scripts/icons-generation/index.mjs",
1010
"generate-illustrations": "rimraf ./illustrations && rimraf ./illustrations-tnt && node ./build-scripts/illustrations-generation/index.mjs",
11-
"generate-local-env": "yarn generate-api-reference && yarn generate-documentation && yarn generate-icons && yarn generate-illustrations",
11+
"generate-llms-txt": "node ./build-scripts/llms-txt-generation/index.mjs",
12+
"generate-local-env": "yarn generate-api-reference && yarn generate-documentation && yarn generate-icons && yarn generate-illustrations && yarn generate-llms-txt",
1213
"generate-production-env": "yarn generate-local-env && rimraf ./static/pages && rimraf ./static/assets && yarn copy:pages:compat && yarn copy:pages:ai && yarn copy:pages:fiori && yarn copy:pages:main",
1314
"docusaurus": "docusaurus",
1415
"start": "yarn generate-local-cdn && yarn generate-local-env && docusaurus start",

0 commit comments

Comments
 (0)